vo_opengl: support external user hooks

This allows users to add their own near-arbitrary hooks to the vo_opengl
processing pipeline, greatly enhancing the flexibility of user shaders.
This enables, among other things, user shaders such as CrossBilateral,
SuperRes, LumaSharpen and many more.

To make parsing the user shaders easier, shaders are now loaded as
bstrs, and the hooks are set up during video reconfig instead of on
every single frame.
This commit is contained in:
Niklas Haas 2016-04-21 01:33:13 +02:00
parent d53142f9ba
commit 7c3d78fd82
8 changed files with 379 additions and 33 deletions

View File

@ -700,6 +700,119 @@ Available video output drivers are:
return vec4(1.0 - color.rgb, color.a);
}
``user-shaders=<files>``
Custom GLSL hooks. These are similar to ``post-shaders`` etc., but more
flexible: They can be injected at almost arbitrary points in the
rendering pipeline, and access all previous intermediate textures.
The general syntax of a user shader looks like this::
//!METADATA ARGS...
//!METADATA ARGS...
vec4 hook() {
...
return something;
}
//!METADATA ARGS...
//!METADATA ARGS...
...
Each block of metadata, along with the non-metadata lines after it,
defines a single pass. Each pass can set the following metadata:
HOOK <name> (required)
The texture which to hook into. May occur multiple times within a
metadata block, up to a predetermined limit. See below for a list
of hookable textures.
BIND <name>
Loads a texture and makes it available to the pass, and sets up
macros to enable accessing it. See below for a list of set macros.
By default, no textures are bound. The special name HOOKED can be
used to refer to the texture that triggered this pass.
SAVE <name>
Gives the name of the texture to save the result of this pass
into. By default, this is set to the special name HOOKED which has
the effect of overwriting the hooked texture.
TRANSFORM sx sy ox oy
Specifies how this pass intends to transform the hooked texture.
``sx``/``sy`` refer to a linear scale factor, and ``ox``/``oy``
refer to a constant pixel shift that the shader will introduce. The
default values are 1 1 0 0 which leave the texture size unchanged.
COMPONENTS n
Specifies how many components of this pass's output are relevant
and should be stored in the texture, up to 4 (rgba). By default,
this value is equal to the number of components in HOOKED.
Each bound texture (via ``BIND``) will make available the following
definitions to that shader pass, where NAME is the name of the bound
texture:
sampler NAME
The bound texture itself.
vec2 NAME_pos
The local texture coordinate of that texture, range [0,1].
vec2 NAME_size
The (rotated) size in pixels of the texture.
vec2 NAME_pt
The (unrotated) size of a single pixel, range [0,1].
In addition, the global uniforms described in ``post-shaders`` are
also available.
Internally, vo_opengl may generate any number of the following
textures. Whenever a texture is rendered and saved by vo_opengl, all of
the passes that have hooked into it will run, in the order they were
added by the user. This is a list of the legal hook points:
RGB, LUMA, CHROMA, ALPHA, XYZ (resizable)
Source planes (raw). Which of these fire depends on the image
format of the source.
CHROMA_SCALED, ALPHA_SCALED (fixed)
Source planes (upscaled). These only fire on subsampled content.
NATIVE (resizable)
The combined image, in the source colorspace, before conversion
to RGB.
MAINPRESUB (resizable)
The image, after conversion to RGB, but before
``blend-subtitles=video`` is applied.
MAIN (resizable)
The main image, after conversion to RGB but before upscaling.
LINEAR (fixed)
Linear light image, before scaling. This only fires when
``linear-scaling`` is in effect.
SIGMOID (fixed)
Sigmoidized light, before scaling. This only fires when
``sigmoid-upscaling`` is in effect.
PREKERNEL (fixed)
The image immediately before the scaler kernel runs.
POSTKERNEL (fixed)
The image immediately after the scaler kernel runs.
SCALED (fixed)
The final upscaled image, before color management.
OUTPUT (fixed)
The final output image, after color management but before
dithering and drawing to screen.
Only the textures labelled with (resizable) may be transformed by
the pass. For all others, the TRANSFORM must be 1 1 0 0 (default).
``deband``
Enable the debanding algorithm. This greatly reduces the amount of
visible banding, blocking and other quantization artifacts, at the

View File

@ -0,0 +1,106 @@
/*
* This file is part of mpv.
*
* mpv is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* mpv is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with mpv. If not, see <http://www.gnu.org/licenses/>.
*/
#include "user_shaders.h"
// Returns false if no more shaders could be parsed
bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
struct gl_user_shader *out)
{
if (!body || !out || !body->start || body->len == 0)
return false;
*out = (struct gl_user_shader){ .transform = identity_trans };
int hook_idx = 0;
int bind_idx = 0;
// First parse all the headers
while (true) {
struct bstr rest;
struct bstr line = bstr_getline(*body, &rest);
// Check for the presence of the magic line beginning
if (!bstr_eatstart0(&line, "//!"))
break;
*body = rest;
// Parse the supported commands
if (bstr_eatstart0(&line, "HOOK")) {
if (hook_idx == SHADER_MAX_HOOKS) {
mp_err(log, "Passes may only hook up to %d textures!\n",
SHADER_MAX_HOOKS);
return false;
}
out->hook_tex[hook_idx++] = bstr_strip(line);
continue;
}
if (bstr_eatstart0(&line, "BIND")) {
if (bind_idx == SHADER_MAX_BINDS) {
mp_err(log, "Passes may only bind up to %d textures!\n",
SHADER_MAX_BINDS);
return false;
}
out->bind_tex[bind_idx++] = bstr_strip(line);
continue;
}
if (bstr_eatstart0(&line, "SAVE")) {
out->save_tex = bstr_strip(line);
continue;
}
if (bstr_eatstart0(&line, "TRANSFORM")) {
float sx, sy, ox, oy;
if (bstr_sscanf(line, "%f %f %f %f", &sx, &sy, &ox, &oy) != 4) {
mp_err(log, "Error while parsing TRANSFORM!\n");
return false;
}
out->transform = (struct gl_transform){{{sx, 0}, {0, sy}}, {ox, oy}};
continue;
}
if (bstr_eatstart0(&line, "COMPONENTS")) {
if (bstr_sscanf(line, "%d", &out->components) != 1) {
mp_err(log, "Error while parsing COMPONENTS!\n");
return false;
}
continue;
}
// Unknown command type
char *str = bstrto0(NULL, line);
mp_err(log, "Unrecognized command '%s'!\n", str);
talloc_free(str);
return false;
}
// The rest of the file up until the next magic line beginning (if any)
// shall be the shader body
if (bstr_split_tok(*body, "//!", &out->pass_body, body)) {
// Make sure the magic line is part of the rest
body->start -= 3;
body->len += 3;
}
// Sanity checking
if (hook_idx == 0)
mp_warn(log, "Pass has no hooked textures (will be ignored)!\n");
return true;
}

View File

@ -0,0 +1,42 @@
/*
* This file is part of mpv.
*
* mpv is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* mpv is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with mpv. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef MP_GL_USER_SHADERS_H
#define MP_GL_USER_SHADERS_H
#include "common.h"
#include "utils.h"
#define SHADER_API 1
#define SHADER_MAX_HOOKS 16
#define SHADER_MAX_BINDS 6
struct gl_user_shader {
struct bstr hook_tex[SHADER_MAX_HOOKS];
struct bstr bind_tex[SHADER_MAX_BINDS];
struct bstr save_tex;
struct bstr pass_body;
struct gl_transform transform;
int components;
};
// Parse the next shader pass from 'body'. Returns false if the end of the
// string was reached
bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
struct gl_user_shader *out);
#endif

View File

@ -565,6 +565,11 @@ void gl_sc_haddf(struct gl_shader_cache *sc, const char *textf, ...)
va_end(ap);
}
void gl_sc_hadd_bstr(struct gl_shader_cache *sc, struct bstr text)
{
bstr_xappend(sc, &sc->header_text, text);
}
static struct sc_uniform *find_uniform(struct gl_shader_cache *sc,
const char *name)
{

View File

@ -152,6 +152,7 @@ void gl_sc_add(struct gl_shader_cache *sc, const char *text);
void gl_sc_addf(struct gl_shader_cache *sc, const char *textf, ...);
void gl_sc_hadd(struct gl_shader_cache *sc, const char *text);
void gl_sc_haddf(struct gl_shader_cache *sc, const char *textf, ...);
void gl_sc_hadd_bstr(struct gl_shader_cache *sc, struct bstr text);
void gl_sc_uniform_sampler(struct gl_shader_cache *sc, char *name, GLenum target,
int unit);
void gl_sc_uniform_sampler_ui(struct gl_shader_cache *sc, char *name, int unit);

View File

@ -40,6 +40,7 @@
#include "superxbr.h"
#include "nnedi3.h"
#include "video_shaders.h"
#include "user_shaders.h"
#include "video/out/filter_kernels.h"
#include "video/out/aspect.h"
#include "video/out/bitmap_packer.h"
@ -152,6 +153,7 @@ struct tex_hook {
void *priv; // this can be set to whatever the hook wants
void (*hook)(struct gl_video *p, struct img_tex tex, // generates GLSL
struct gl_transform *trans, void *priv);
void (*free)(struct tex_hook *hook);
};
struct fbosurface {
@ -163,7 +165,7 @@ struct fbosurface {
struct cached_file {
char *path;
char *body;
struct bstr body;
};
struct gl_video {
@ -424,6 +426,7 @@ const struct m_sub_options gl_video_conf = {
OPT_STRING("scale-shader", scale_shader, 0),
OPT_STRINGLIST("pre-shaders", pre_shaders, 0),
OPT_STRINGLIST("post-shaders", post_shaders, 0),
OPT_STRINGLIST("user-shaders", user_shaders, 0),
OPT_FLAG("deband", deband, 0),
OPT_SUBSTRUCT("deband", deband_opts, deband_conf, 0),
OPT_FLOAT("sharpen", unsharp, 0),
@ -483,10 +486,10 @@ static void get_scale_factors(struct gl_video *p, bool transpose_rot, double xy[
#define GLSLF(...) gl_sc_addf(p->sc, __VA_ARGS__)
#define GLSLHF(...) gl_sc_haddf(p->sc, __VA_ARGS__)
static const char *load_cached_file(struct gl_video *p, const char *path)
static struct bstr load_cached_file(struct gl_video *p, const char *path)
{
if (!path || !path[0])
return NULL;
return (struct bstr){0};
for (int n = 0; n < p->num_files; n++) {
if (strcmp(p->files[n].path, path) == 0)
return p->files[n].body;
@ -496,7 +499,7 @@ static const char *load_cached_file(struct gl_video *p, const char *path)
// empty cache when it overflows
for (int n = 0; n < p->num_files; n++) {
talloc_free(p->files[n].path);
talloc_free(p->files[n].body);
talloc_free(p->files[n].body.start);
}
p->num_files = 0;
}
@ -505,11 +508,11 @@ static const char *load_cached_file(struct gl_video *p, const char *path)
struct cached_file *new = &p->files[p->num_files++];
*new = (struct cached_file) {
.path = talloc_strdup(p, path),
.body = s.start
.body = s,
};
return new->body;
}
return NULL;
return (struct bstr){0};
}
static void debug_check_gl(struct gl_video *p, const char *msg)
@ -537,6 +540,16 @@ static void gl_video_reset_surfaces(struct gl_video *p)
p->output_fbo_valid = false;
}
static void gl_video_reset_hooks(struct gl_video *p)
{
for (int i = 0; i < p->tex_hook_num; i++) {
if (p->tex_hooks[i].free)
p->tex_hooks[i].free(&p->tex_hooks[i]);
}
p->tex_hook_num = 0;
}
static inline int fbosurface_wrap(int id)
{
id = id % FBOSURFACES_MAX;
@ -553,6 +566,7 @@ static void recreate_osd(struct gl_video *p)
}
}
static void gl_video_setup_hooks(struct gl_video *p);
static void reinit_rendering(struct gl_video *p)
{
MP_VERBOSE(p, "Reinit rendering.\n");
@ -562,6 +576,8 @@ static void reinit_rendering(struct gl_video *p)
uninit_rendering(p);
recreate_osd(p);
gl_video_setup_hooks(p);
}
static void uninit_rendering(struct gl_video *p)
@ -596,6 +612,7 @@ static void uninit_rendering(struct gl_video *p)
fbotex_uninit(&p->hook_fbos[n]);
gl_video_reset_surfaces(p);
gl_video_reset_hooks(p);
}
void gl_video_update_profile(struct gl_video *p)
@ -1167,19 +1184,26 @@ static void pass_opt_hook_point(struct gl_video *p, const char *name,
if (!name)
return;
int i;
for (i = 0; i < p->tex_hook_num; i++) {
if (strcmp(p->tex_hooks[i].hook_tex, name) == 0)
break;
for (int i = 0; i < p->tex_hook_num; i++) {
struct tex_hook *hook = &p->tex_hooks[i];
if (strcmp(hook->hook_tex, name) == 0)
goto found;
for (int b = 0; b < TEXUNIT_VIDEO_NUM; b++) {
if (hook->bind_tex[b] && strcmp(hook->bind_tex[b], name) == 0)
goto found;
}
}
if (i == p->tex_hook_num)
return;
// Nothing uses this texture, don't bother storing it
return;
found:
assert(p->hook_fbo_num < MAX_SAVED_TEXTURES);
struct fbotex *fbo = &p->hook_fbos[p->hook_fbo_num++];
finish_pass_fbo(p, fbo, p->texture_w, p->texture_h, 0);
struct img_tex img = img_tex_fbo(fbo, PLANE_RGB, p->components);
img = pass_hook(p, name, img, tex_trans);
copy_img_tex(p, &(int){0}, img);
@ -1188,9 +1212,9 @@ static void pass_opt_hook_point(struct gl_video *p, const char *name,
p->components = img.components;
}
static void load_shader(struct gl_video *p, const char *body)
static void load_shader(struct gl_video *p, struct bstr body)
{
gl_sc_hadd(p->sc, body);
gl_sc_hadd_bstr(p->sc, body);
gl_sc_uniform_f(p->sc, "random", (double)av_lfg_get(&p->lfg) / UINT32_MAX);
gl_sc_uniform_f(p->sc, "frame", p->frames_uploaded);
gl_sc_uniform_vec2(p->sc, "image_size", (GLfloat[]){p->image_params.w,
@ -1400,10 +1424,10 @@ static void pass_sample(struct gl_video *p, struct img_tex tex,
} else if (strcmp(name, "oversample") == 0) {
pass_sample_oversample(p->sc, scaler, w, h);
} else if (strcmp(name, "custom") == 0) {
const char *body = load_cached_file(p, p->opts.scale_shader);
if (body) {
struct bstr body = load_cached_file(p, p->opts.scale_shader);
if (body.start) {
load_shader(p, body);
const char *fn_name = get_custom_shader_fn(p, body);
const char *fn_name = get_custom_shader_fn(p, body.start);
GLSLF("// custom scale-shader\n");
GLSLF("color = %s(tex, pos, size);\n", fn_name);
} else {
@ -1581,45 +1605,95 @@ static void unsharp_hook(struct gl_video *p, struct img_tex tex,
pass_sample_unsharp(p->sc, p->opts.unsharp);
}
static void user_shader_hook(struct gl_video *p, struct img_tex tex,
struct gl_transform *trans, void *priv)
static void user_hook_old(struct gl_video *p, struct img_tex tex,
struct gl_transform *trans, void *priv)
{
const char *body = priv;
assert(body);
GLSLHF("#define pixel_size HOOKED_pt\n");
load_shader(p, body);
load_shader(p, bstr0(body));
const char *fn_name = get_custom_shader_fn(p, body);
GLSLF("// custom shader\n");
GLSLF("color = %s(HOOKED, HOOKED_pos, HOOKED_size);\n", fn_name);
}
static void pass_hook_user_shaders(struct gl_video *p, const char *name,
char **shaders)
static void user_hook(struct gl_video *p, struct img_tex tex,
struct gl_transform *trans, void *priv)
{
struct gl_user_shader *shader = priv;
assert(shader);
load_shader(p, shader->pass_body);
GLSLF("// custom hook\n");
GLSLF("color = hook();\n");
*trans = shader->transform;
}
static void user_hook_free(struct tex_hook *hook)
{
talloc_free((void *)hook->hook_tex);
talloc_free((void *)hook->save_tex);
for (int i = 0; i < TEXUNIT_VIDEO_NUM; i++)
talloc_free((void *)hook->bind_tex[i]);
talloc_free(hook->priv);
}
static void pass_hook_user_shaders_old(struct gl_video *p, const char *name,
char **shaders)
{
assert(name);
if (!shaders)
return;
for (int n = 0; shaders[n] != NULL; n++) {
const char *body = load_cached_file(p, shaders[n]);
const char *body = load_cached_file(p, shaders[n]).start;
if (body) {
pass_add_hook(p, (struct tex_hook) {
.hook_tex = name,
.bind_tex = {"HOOKED"},
.hook = user_shader_hook,
.hook = user_hook_old,
.priv = (void *)body,
});
}
}
}
static void pass_setup_hooks(struct gl_video *p)
static void pass_hook_user_shaders(struct gl_video *p, char **shaders)
{
// Reset any existing hooks
p->tex_hook_num = 0;
memset(&p->tex_hooks, 0, sizeof(p->tex_hooks));
if (!shaders)
return;
for (int n = 0; shaders[n] != NULL; n++) {
struct bstr file = load_cached_file(p, shaders[n]);
struct gl_user_shader out;
while (parse_user_shader_pass(p->log, &file, &out)) {
struct tex_hook hook = {
.components = out.components,
.hook = user_hook,
.free = user_hook_free,
};
for (int i = 0; i < SHADER_MAX_HOOKS; i++) {
hook.hook_tex = bstrdup0(p, out.hook_tex[i]);
if (!hook.hook_tex)
continue;
struct gl_user_shader *out_copy = talloc_ptrtype(p, out_copy);
*out_copy = out;
hook.priv = out_copy;
for (int o = 0; o < SHADER_MAX_BINDS; o++)
hook.bind_tex[o] = bstrdup0(p, out.bind_tex[o]);
hook.save_tex = bstrdup0(p, out.save_tex),
pass_add_hook(p, hook);
}
}
}
}
static void gl_video_setup_hooks(struct gl_video *p)
{
if (p->opts.deband) {
pass_add_hooks(p, (struct tex_hook) {.hook = deband_hook},
HOOKS("LUMA", "CHROMA", "RGB", "XYZ"));
@ -1642,8 +1716,9 @@ static void pass_setup_hooks(struct gl_video *p)
});
}
pass_hook_user_shaders(p, "MAIN", p->opts.pre_shaders);
pass_hook_user_shaders(p, "SCALED", p->opts.post_shaders);
pass_hook_user_shaders_old(p, "MAIN", p->opts.pre_shaders);
pass_hook_user_shaders_old(p, "SCALED", p->opts.post_shaders);
pass_hook_user_shaders(p, p->opts.user_shaders);
}
// sample from video textures, set "color" variable to yuv value
@ -2278,8 +2353,6 @@ static void pass_render_frame(struct gl_video *p)
if (p->dumb_mode)
return;
pass_setup_hooks(p);
p->use_linear = p->opts.linear_scaling || p->opts.sigmoid_upscaling;
pass_read_video(p);
pass_opt_hook_point(p, "NATIVE", &p->texture_offset);
@ -2805,6 +2878,8 @@ static bool check_dumb_mode(struct gl_video *p)
return false;
if (o->post_shaders && o->post_shaders[0])
return false;
if (o->user_shaders && o->user_shaders[0])
return false;
if (p->use_lut_3d)
return false;
return true;
@ -3275,6 +3350,7 @@ static void assign_options(struct gl_video_opts *dst, struct gl_video_opts *src)
talloc_free(dst->scale_shader);
talloc_free(dst->pre_shaders);
talloc_free(dst->post_shaders);
talloc_free(dst->user_shaders);
talloc_free(dst->deband_opts);
talloc_free(dst->superxbr_opts);
talloc_free(dst->nnedi3_opts);
@ -3303,6 +3379,7 @@ static void assign_options(struct gl_video_opts *dst, struct gl_video_opts *src)
dst->scale_shader = talloc_strdup(NULL, dst->scale_shader);
dst->pre_shaders = dup_str_array(NULL, dst->pre_shaders);
dst->post_shaders = dup_str_array(NULL, dst->post_shaders);
dst->user_shaders = dup_str_array(NULL, dst->user_shaders);
}
// Set the options, and possibly update the filter chain too.

View File

@ -108,6 +108,7 @@ struct gl_video_opts {
char *scale_shader;
char **pre_shaders;
char **post_shaders;
char **user_shaders;
int deband;
struct deband_opts *deband_opts;
float unsharp;

View File

@ -356,6 +356,7 @@ def build(ctx):
( "video/out/opengl/nnedi3.c", "gl" ),
( "video/out/opengl/osd.c", "gl" ),
( "video/out/opengl/superxbr.c", "gl" ),
( "video/out/opengl/user_shaders.c", "gl" ),
( "video/out/opengl/utils.c", "gl" ),
( "video/out/opengl/video.c", "gl" ),
( "video/out/opengl/video_shaders.c", "gl" ),