mpv/video/out/vo_opengl_cb.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdbool.h>
#include <limits.h>
#include <pthread.h>
#include <assert.h>

#include "config.h"

#include "mpv_talloc.h"
#include "common/common.h"
#include "misc/bstr.h"
#include "common/msg.h"
#include "options/m_config.h"
#include "options/options.h"
#include "aspect.h"
#include "vo.h"
#include "video/mp_image.h"
#include "sub/osd.h"
#include "osdep/timer.h"

#include "common/global.h"
#include "player/client.h"

#include "gpu/video.h"
#include "gpu/hwdec.h"
#include "opengl/common.h"
#include "opengl/context.h"
#include "opengl/ra_gl.h"

#include "libmpv/opengl_cb.h"

/*
 * mpv_opengl_cb_context is created by the host application - the host application
 * can access it any time, even if the VO is destroyed (or not created yet).
 * The OpenGL object allows initializing the renderer etc. The VO object is only
 * here to transfer the video frames somehow.
 *
 * Locking hierarchy:
 * - the libmpv user can mix openglcb and normal API; thus openglcb API
 *   functions can wait on the core, but not the reverse
 * - the core does blocking calls into the VO thread, thus the VO functions
 *   can't wait on the user calling the API functions
 * - to make video timing work like it should, the VO thread waits on the
 *   openglcb API user anyway, and the (unlikely) deadlock is avoided with
 *   a timeout
 */

struct vo_priv {
    struct mpv_opengl_cb_context *ctx;
};

struct mpv_opengl_cb_context {
    struct mp_log *log;
    struct mpv_global *global;
    struct mp_client_api *client_api;

    pthread_mutex_t lock;
    pthread_cond_t wakeup;

    // --- Protected by lock
    bool initialized;
    mpv_opengl_cb_update_fn update_cb;
    void *update_cb_ctx;
    struct vo_frame *next_frame;    // next frame to draw
    int64_t present_count;          // incremented when next frame can be shown
    int64_t expected_flip_count;    // next vsync event for next_frame
    bool redrawing;                 // next_frame was a redraw request
    int64_t flip_count;
    struct vo_frame *cur_frame;
    struct mp_image_params img_params;
    bool reconfigured, reset;
    int vp_w, vp_h;
    bool flip;
    bool force_update;
    bool imgfmt_supported[IMGFMT_END - IMGFMT_START];
    bool update_new_opts;
    struct vo *active;

    // --- This is only mutable while initialized=false, during which nothing
    //     except the OpenGL context manager is allowed to access it.
    struct mp_hwdec_devices *hwdec_devs;

    // --- All of these can only be accessed from the thread where the host
    //     application's OpenGL context is current - i.e. only while the
    //     host application is calling certain mpv_opengl_cb_* APIs.
    GL *gl;
    struct ra_ctx *ra_ctx;
    struct gl_video *renderer;
    struct ra_hwdec *hwdec;
    struct m_config_cache *vo_opts_cache;
    struct mp_vo_opts *vo_opts;
};

static void update(struct vo_priv *p);

static void forget_frames(struct mpv_opengl_cb_context *ctx, bool all)
{
    pthread_cond_broadcast(&ctx->wakeup);
    if (all) {
        talloc_free(ctx->cur_frame);
        ctx->cur_frame = NULL;
    }
}

static void free_ctx(void *ptr)
{
    mpv_opengl_cb_context *ctx = ptr;

    // This can trigger if the client API user doesn't call
    // mpv_opengl_cb_uninit_gl() properly.
    assert(!ctx->initialized);

    pthread_cond_destroy(&ctx->wakeup);
    pthread_mutex_destroy(&ctx->lock);
}

struct mpv_opengl_cb_context *mp_opengl_create(struct mpv_global *g,
                                               struct mp_client_api *client_api)
{
    mpv_opengl_cb_context *ctx = talloc_zero(NULL, mpv_opengl_cb_context);
    talloc_set_destructor(ctx, free_ctx);
    pthread_mutex_init(&ctx->lock, NULL);
    pthread_cond_init(&ctx->wakeup, NULL);

    ctx->global = g;
    ctx->log = mp_log_new(ctx, g->log, "opengl-cb");
    ctx->client_api = client_api;

    ctx->vo_opts_cache = m_config_cache_alloc(ctx, ctx->global, &vo_sub_opts);
    ctx->vo_opts = ctx->vo_opts_cache->opts;

    return ctx;
}

void mpv_opengl_cb_set_update_callback(struct mpv_opengl_cb_context *ctx,
                                      mpv_opengl_cb_update_fn callback,
                                      void *callback_ctx)
{
    pthread_mutex_lock(&ctx->lock);
    ctx->update_cb = callback;
    ctx->update_cb_ctx = callback_ctx;
    pthread_mutex_unlock(&ctx->lock);
}

// Reset some GL attributes the user might clobber. For mid-term compatibility
// only - we expect both user code and our code to do this correctly.
static void reset_gl_state(GL *gl)
{
    gl->ActiveTexture(GL_TEXTURE0);
    if (gl->mpgl_caps & MPGL_CAP_ROW_LENGTH)
        gl->PixelStorei(GL_UNPACK_ROW_LENGTH, 0);
    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
}

int mpv_opengl_cb_init_gl(struct mpv_opengl_cb_context *ctx, const char *exts,
                          mpv_opengl_cb_get_proc_address_fn get_proc_address,
                          void *get_proc_address_ctx)
{
    if (ctx->renderer)
        return MPV_ERROR_INVALID_PARAMETER;

    talloc_free(ctx->gl);
    ctx->gl = talloc_zero(ctx, GL);

    mpgl_load_functions2(ctx->gl, get_proc_address, get_proc_address_ctx,
                         exts, ctx->log);
    if (!ctx->gl->version && !ctx->gl->es) {
        MP_FATAL(ctx, "OpenGL not initialized.\n");
        return MPV_ERROR_UNSUPPORTED;
    }

    // initialize a blank ra_ctx to reuse ra_gl_ctx
    ctx->ra_ctx = talloc_zero(ctx, struct ra_ctx);
    ctx->ra_ctx->log = ctx->log;
    ctx->ra_ctx->global = ctx->global;
    ctx->ra_ctx->opts = (struct ra_ctx_opts) {
        .probing = false,
        .allow_sw = true,
    };

    static const struct ra_swapchain_fns empty_swapchain_fns = {0};
    struct ra_gl_ctx_params gl_params = {
        // vo_opengl_cb is essentially like a gigantic external swapchain where
        // the user is in charge of presentation / swapping etc. But we don't
        // actually need to provide any of these functions, since we can just
        // not call them to begin with - so just set it to an empty object to
        // signal to ra_gl_ctx that we don't care about its latency emulation
        // functionality
        .external_swapchain = &empty_swapchain_fns
    };

    ctx->gl->SwapInterval = NULL; // we shouldn't randomly change this, so lock it
    if (!ra_gl_ctx_init(ctx->ra_ctx, ctx->gl, gl_params))
        return MPV_ERROR_UNSUPPORTED;

    ctx->renderer = gl_video_init(ctx->ra_ctx->ra, ctx->log, ctx->global);

    m_config_cache_update(ctx->vo_opts_cache);

    ctx->hwdec_devs = hwdec_devices_create();
    ctx->hwdec = ra_hwdec_load(ctx->log, ctx->ra_ctx->ra, ctx->global,
                               ctx->hwdec_devs, ctx->vo_opts->gl_hwdec_interop);
    gl_video_set_hwdec(ctx->renderer, ctx->hwdec);

    pthread_mutex_lock(&ctx->lock);
    for (int n = IMGFMT_START; n < IMGFMT_END; n++) {
        ctx->imgfmt_supported[n - IMGFMT_START] =
            gl_video_check_format(ctx->renderer, n);
    }
    ctx->initialized = true;
    pthread_mutex_unlock(&ctx->lock);

    reset_gl_state(ctx->gl);
    return 0;
}

int mpv_opengl_cb_uninit_gl(struct mpv_opengl_cb_context *ctx)
{
    if (!ctx)
        return 0;

    // Bring down the decoder etc., which still might be using the hwdec
    // context. Setting initialized=false guarantees it can't come back.

    pthread_mutex_lock(&ctx->lock);
    forget_frames(ctx, true);
    ctx->initialized = false;
    pthread_mutex_unlock(&ctx->lock);

    kill_video(ctx->client_api);

    pthread_mutex_lock(&ctx->lock);
    assert(!ctx->active);
    pthread_mutex_unlock(&ctx->lock);

    gl_video_uninit(ctx->renderer);
    ctx->renderer = NULL;
    ra_hwdec_uninit(ctx->hwdec);
    ctx->hwdec = NULL;
    hwdec_devices_destroy(ctx->hwdec_devs);
    ctx->hwdec_devs = NULL;
    ra_gl_ctx_uninit(ctx->ra_ctx);
    talloc_free(ctx->ra_ctx);
    talloc_free(ctx->gl);
    ctx->ra_ctx = NULL;
    ctx->gl = NULL;
    return 0;
}

int mpv_opengl_cb_draw(mpv_opengl_cb_context *ctx, int fbo, int vp_w, int vp_h)
{
    assert(ctx->renderer);

    if (fbo && !(ctx->gl->mpgl_caps & MPGL_CAP_FB)) {
        MP_FATAL(ctx, "Rendering to FBO requested, but no FBO extension found!\n");
        return MPV_ERROR_UNSUPPORTED;
    }

    reset_gl_state(ctx->gl);

    pthread_mutex_lock(&ctx->lock);

    struct vo *vo = ctx->active;

    ctx->force_update |= ctx->reconfigured;

    if (ctx->vp_w != vp_w || ctx->vp_h != vp_h)
        ctx->force_update = true;

    if (ctx->force_update && vo) {
        ctx->force_update = false;
        ctx->vp_w = vp_w;
        ctx->vp_h = vp_h;

        m_config_cache_update(ctx->vo_opts_cache);

        struct mp_rect src, dst;
        struct mp_osd_res osd;
        mp_get_src_dst_rects(ctx->log, ctx->vo_opts, vo->driver->caps,
                             &ctx->img_params, vp_w, abs(vp_h),
                             1.0, &src, &dst, &osd);

        gl_video_resize(ctx->renderer, &src, &dst, &osd);
    }

    if (ctx->reconfigured) {
        gl_video_set_osd_source(ctx->renderer, vo ? vo->osd : NULL);
        gl_video_config(ctx->renderer, &ctx->img_params);
    }
    if (ctx->update_new_opts) {
        if (vo)
            gl_video_configure_queue(ctx->renderer, vo);
        int debug;
        mp_read_option_raw(ctx->global, "gpu-debug", &m_option_type_flag,
                           &debug);
        ctx->gl->debug_context = debug;
        ra_gl_set_debug(ctx->ra_ctx->ra, debug);
        if (gl_video_icc_auto_enabled(ctx->renderer))
            MP_ERR(ctx, "icc-profile-auto is not available with opengl-cb\n");
    }
    ctx->reconfigured = false;
    ctx->update_new_opts = false;

    if (ctx->reset) {
        gl_video_reset(ctx->renderer);
        ctx->reset = false;
        if (ctx->cur_frame)
            ctx->cur_frame->still = true;
    }

    struct vo_frame *frame = ctx->next_frame;
    int64_t wait_present_count = ctx->present_count;
    if (frame) {
        ctx->next_frame = NULL;
        if (!(frame->redraw || !frame->current))
            wait_present_count += 1;
        pthread_cond_signal(&ctx->wakeup);
        talloc_free(ctx->cur_frame);
        ctx->cur_frame = vo_frame_ref(frame);
    } else {
        frame = vo_frame_ref(ctx->cur_frame);
        if (frame)
            frame->redraw = true;
        MP_STATS(ctx, "glcb-noframe");
    }
    struct vo_frame dummy = {0};
    if (!frame)
        frame = &dummy;

    pthread_mutex_unlock(&ctx->lock);

    MP_STATS(ctx, "glcb-render");
    struct ra_swapchain *sw = ctx->ra_ctx->swapchain;
    struct ra_fbo target;
    ra_gl_ctx_resize(sw, vp_w, abs(vp_h), fbo);
    ra_gl_ctx_start_frame(sw, &target);
    target.flip = vp_h < 0;
    gl_video_render_frame(ctx->renderer, frame, target);
    ra_gl_ctx_submit_frame(sw, frame);

    reset_gl_state(ctx->gl);

    if (frame != &dummy)
        talloc_free(frame);

    pthread_mutex_lock(&ctx->lock);
    while (wait_present_count > ctx->present_count)
        pthread_cond_wait(&ctx->wakeup, &ctx->lock);
    pthread_mutex_unlock(&ctx->lock);

    return 0;
}

int mpv_opengl_cb_report_flip(mpv_opengl_cb_context *ctx, int64_t time)
{
    MP_STATS(ctx, "glcb-reportflip");

    pthread_mutex_lock(&ctx->lock);
    ctx->flip_count += 1;
    pthread_cond_signal(&ctx->wakeup);
    pthread_mutex_unlock(&ctx->lock);

    return 0;
}

// Called locked.
static void update(struct vo_priv *p)
{
    if (p->ctx->update_cb)
        p->ctx->update_cb(p->ctx->update_cb_ctx);
}

static void draw_frame(struct vo *vo, struct vo_frame *frame)
{
    struct vo_priv *p = vo->priv;

    pthread_mutex_lock(&p->ctx->lock);
    assert(!p->ctx->next_frame);
    p->ctx->next_frame = vo_frame_ref(frame);
    p->ctx->expected_flip_count = p->ctx->flip_count + 1;
    p->ctx->redrawing = frame->redraw || !frame->current;
    update(p);
    pthread_mutex_unlock(&p->ctx->lock);
}

static void flip_page(struct vo *vo)
{
    struct vo_priv *p = vo->priv;
    struct timespec ts = mp_rel_time_to_timespec(0.2);

    pthread_mutex_lock(&p->ctx->lock);

    // Wait until frame was rendered
    while (p->ctx->next_frame) {
        if (pthread_cond_timedwait(&p->ctx->wakeup, &p->ctx->lock, &ts)) {
            if (p->ctx->next_frame) {
                MP_VERBOSE(vo, "mpv_opengl_cb_draw() not being called or stuck.\n");
                goto done;
            }
        }
    }

    // Unblock mpv_opengl_cb_draw().
    p->ctx->present_count += 1;
    pthread_cond_signal(&p->ctx->wakeup);

    if (p->ctx->redrawing)
        goto done; // do not block for redrawing

    // Wait until frame was presented
    while (p->ctx->expected_flip_count > p->ctx->flip_count) {
        // mpv_opengl_cb_report_flip() is declared as optional API.
        // Assume the user calls it consistently _if_ it's called at all.
        if (!p->ctx->flip_count)
            break;
        if (pthread_cond_timedwait(&p->ctx->wakeup, &p->ctx->lock, &ts)) {
            MP_VERBOSE(vo, "mpv_opengl_cb_report_flip() not being called.\n");
            goto done;
        }
    }

done:

    // Cleanup after the API user is not reacting, or is being unusually slow.
    if (p->ctx->next_frame) {
        talloc_free(p->ctx->cur_frame);
        p->ctx->cur_frame = p->ctx->next_frame;
        p->ctx->next_frame = NULL;
        p->ctx->present_count += 2;
        pthread_cond_signal(&p->ctx->wakeup);
        vo_increment_drop_count(vo, 1);
    }

    pthread_mutex_unlock(&p->ctx->lock);
}

static int query_format(struct vo *vo, int format)
{
    struct vo_priv *p = vo->priv;

    bool ok = false;
    pthread_mutex_lock(&p->ctx->lock);
    if (format >= IMGFMT_START && format < IMGFMT_END)
        ok = p->ctx->imgfmt_supported[format - IMGFMT_START];
    pthread_mutex_unlock(&p->ctx->lock);
    return ok;
}

static int reconfig(struct vo *vo, struct mp_image_params *params)
{
    struct vo_priv *p = vo->priv;

    pthread_mutex_lock(&p->ctx->lock);
    forget_frames(p->ctx, true);
    p->ctx->img_params = *params;
    p->ctx->reconfigured = true;
    pthread_mutex_unlock(&p->ctx->lock);

    return 0;
}

static int control(struct vo *vo, uint32_t request, void *data)
{
    struct vo_priv *p = vo->priv;

    switch (request) {
    case VOCTRL_RESET:
        pthread_mutex_lock(&p->ctx->lock);
        forget_frames(p->ctx, false);
        p->ctx->reset = true;
        pthread_mutex_unlock(&p->ctx->lock);
        return VO_TRUE;
    case VOCTRL_PAUSE:
        vo->want_redraw = true;
        return VO_TRUE;
    case VOCTRL_SET_EQUALIZER:
        vo->want_redraw = true;
        return VO_TRUE;
    case VOCTRL_SET_PANSCAN:
        pthread_mutex_lock(&p->ctx->lock);
        p->ctx->force_update = true;
        update(p);
        pthread_mutex_unlock(&p->ctx->lock);
        return VO_TRUE;
    case VOCTRL_UPDATE_RENDER_OPTS:
        pthread_mutex_lock(&p->ctx->lock);
        p->ctx->update_new_opts = true;
        update(p);
        pthread_mutex_unlock(&p->ctx->lock);
        return VO_TRUE;
    }

    return VO_NOTIMPL;
}

static void uninit(struct vo *vo)
{
    struct vo_priv *p = vo->priv;

    pthread_mutex_lock(&p->ctx->lock);
    forget_frames(p->ctx, true);
    p->ctx->img_params = (struct mp_image_params){0};
    p->ctx->reconfigured = true;
    p->ctx->active = NULL;
    update(p);
    pthread_mutex_unlock(&p->ctx->lock);
}

static int preinit(struct vo *vo)
{
    struct vo_priv *p = vo->priv;
    p->ctx = vo->extra.opengl_cb_context;
    if (!p->ctx) {
        MP_FATAL(vo, "No context set.\n");
        return -1;
    }

    pthread_mutex_lock(&p->ctx->lock);
    if (!p->ctx->initialized) {
        MP_FATAL(vo, "OpenGL context not initialized.\n");
        pthread_mutex_unlock(&p->ctx->lock);
        return -1;
    }
    p->ctx->active = vo;
    p->ctx->reconfigured = true;
    p->ctx->update_new_opts = true;
    pthread_mutex_unlock(&p->ctx->lock);

    vo->hwdec_devs = p->ctx->hwdec_devs;

    return 0;
}

const struct vo_driver video_out_opengl_cb = {
    .description = "OpenGL Callbacks for libmpv",
    .name = "opengl-cb",
    .caps = VO_CAP_ROTATE90,
    .preinit = preinit,
    .query_format = query_format,
    .reconfig = reconfig,
    .control = control,
    .draw_frame = draw_frame,
    .flip_page = flip_page,
    .uninit = uninit,
    .priv_size = sizeof(struct vo_priv),
};