mirror of https://git.ffmpeg.org/ffmpeg.git
fftools/ffmpeg: move sub2video handling to ffmpeg_filter
Make all relevant state per-filtergraph input, rather than per-input stream. Refactor the code to make it work and avoid leaking memory when a single subtitle stream is sent to multiple filters.
This commit is contained in:
Anton Khirnov
parent
20cacfe493
commit
f8abab673c
133
fftools/ffmpeg.c
133
fftools/ffmpeg.c
|
|
@ -147,139 +147,20 @@ static int restore_tty;
|
|||
This is a temporary solution until libavfilter gets real subtitles support.
|
||||
*/
|
||||
|
||||
static int sub2video_get_blank_frame(InputStream *ist)
|
||||
{
|
||||
int ret;
|
||||
AVFrame *frame = ist->sub2video.frame;
|
||||
|
||||
av_frame_unref(frame);
|
||||
frame->width = ist->sub2video.w;
|
||||
frame->height = ist->sub2video.h;
|
||||
frame->format = AV_PIX_FMT_RGB32;
|
||||
if ((ret = av_frame_get_buffer(frame, 0)) < 0)
|
||||
return ret;
|
||||
memset(frame->data[0], 0, frame->height * frame->linesize[0]);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
|
||||
AVSubtitleRect *r)
|
||||
{
|
||||
uint32_t *pal, *dst2;
|
||||
uint8_t *src, *src2;
|
||||
int x, y;
|
||||
|
||||
if (r->type != SUBTITLE_BITMAP) {
|
||||
av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
|
||||
return;
|
||||
}
|
||||
if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
|
||||
av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
|
||||
r->x, r->y, r->w, r->h, w, h
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
dst += r->y * dst_linesize + r->x * 4;
|
||||
src = r->data[0];
|
||||
pal = (uint32_t *)r->data[1];
|
||||
for (y = 0; y < r->h; y++) {
|
||||
dst2 = (uint32_t *)dst;
|
||||
src2 = src;
|
||||
for (x = 0; x < r->w; x++)
|
||||
*(dst2++) = pal[*(src2++)];
|
||||
dst += dst_linesize;
|
||||
src += r->linesize[0];
|
||||
}
|
||||
}
|
||||
|
||||
static void sub2video_push_ref(InputStream *ist, int64_t pts)
|
||||
{
|
||||
AVFrame *frame = ist->sub2video.frame;
|
||||
int i;
|
||||
int ret;
|
||||
|
||||
av_assert1(frame->data[0]);
|
||||
ist->sub2video.last_pts = frame->pts = pts;
|
||||
for (i = 0; i < ist->nb_filters; i++) {
|
||||
ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame,
|
||||
AV_BUFFERSRC_FLAG_KEEP_REF |
|
||||
AV_BUFFERSRC_FLAG_PUSH);
|
||||
if (ret != AVERROR_EOF && ret < 0)
|
||||
av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
|
||||
av_err2str(ret));
|
||||
}
|
||||
}
|
||||
|
||||
void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
|
||||
const AVSubtitle *sub)
|
||||
{
|
||||
AVFrame *frame = ist->sub2video.frame;
|
||||
int8_t *dst;
|
||||
int dst_linesize;
|
||||
int num_rects, i;
|
||||
int64_t pts, end_pts;
|
||||
|
||||
if (!frame)
|
||||
return;
|
||||
if (sub) {
|
||||
pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
|
||||
AV_TIME_BASE_Q, ist->st->time_base);
|
||||
end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL,
|
||||
AV_TIME_BASE_Q, ist->st->time_base);
|
||||
num_rects = sub->num_rects;
|
||||
} else {
|
||||
/* If we are initializing the system, utilize current heartbeat
|
||||
PTS as the start time, and show until the following subpicture
|
||||
is received. Otherwise, utilize the previous subpicture's end time
|
||||
as the fall-back value. */
|
||||
pts = ist->sub2video.initialize ?
|
||||
heartbeat_pts : ist->sub2video.end_pts;
|
||||
end_pts = INT64_MAX;
|
||||
num_rects = 0;
|
||||
}
|
||||
if (sub2video_get_blank_frame(ist) < 0) {
|
||||
av_log(NULL, AV_LOG_ERROR,
|
||||
"Impossible to get a blank canvas.\n");
|
||||
return;
|
||||
}
|
||||
dst = frame->data [0];
|
||||
dst_linesize = frame->linesize[0];
|
||||
for (i = 0; i < num_rects; i++)
|
||||
sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
|
||||
sub2video_push_ref(ist, pts);
|
||||
ist->sub2video.end_pts = end_pts;
|
||||
ist->sub2video.initialize = 0;
|
||||
}
|
||||
|
||||
static void sub2video_heartbeat(InputFile *infile, int64_t pts, AVRational tb)
|
||||
{
|
||||
int i, j, nb_reqs;
|
||||
int64_t pts2;
|
||||
|
||||
/* When a frame is read from a file, examine all sub2video streams in
|
||||
the same file and send the sub2video frame again. Otherwise, decoded
|
||||
video frames could be accumulating in the filter graph while a filter
|
||||
(possibly overlay) is desperately waiting for a subtitle frame. */
|
||||
for (i = 0; i < infile->nb_streams; i++) {
|
||||
InputStream *ist2 = infile->streams[i];
|
||||
if (!ist2->sub2video.frame)
|
||||
for (int i = 0; i < infile->nb_streams; i++) {
|
||||
InputStream *ist = infile->streams[i];
|
||||
|
||||
if (ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
|
||||
continue;
|
||||
/* subtitles seem to be usually muxed ahead of other streams;
|
||||
if not, subtracting a larger time here is necessary */
|
||||
pts2 = av_rescale_q(pts, tb, ist2->st->time_base) - 1;
|
||||
/* do not send the heartbeat frame if the subtitle is already ahead */
|
||||
if (pts2 <= ist2->sub2video.last_pts)
|
||||
continue;
|
||||
if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize)
|
||||
/* if we have hit the end of the current displayed subpicture,
|
||||
or if we need to initialize the system, update the
|
||||
overlayed subpicture and its start/end times */
|
||||
sub2video_update(ist2, pts2 + 1, NULL);
|
||||
for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
|
||||
nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
|
||||
if (nb_reqs)
|
||||
sub2video_push_ref(ist2, pts2);
|
||||
|
||||
for (int j = 0; j < ist->nb_filters; j++)
|
||||
ifilter_sub2video_heartbeat(ist->filters[j], pts, tb);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -372,11 +372,7 @@ typedef struct InputStream {
|
|||
} prev_sub;
|
||||
|
||||
struct sub2video {
|
||||
int64_t last_pts;
|
||||
int64_t end_pts;
|
||||
AVFrame *frame;
|
||||
int w, h;
|
||||
unsigned int initialize; ///< marks if sub2video_update should force an initialization
|
||||
} sub2video;
|
||||
|
||||
/* decoded data from this stream goes into all those filters
|
||||
|
|
@ -741,13 +737,12 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost,
|
|||
char *graph_desc);
|
||||
int init_complex_filtergraph(FilterGraph *fg);
|
||||
|
||||
void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
|
||||
const AVSubtitle *sub);
|
||||
int copy_av_subtitle(AVSubtitle *dst, const AVSubtitle *src);
|
||||
|
||||
int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame, int keep_reference);
|
||||
int ifilter_send_eof(InputFilter *ifilter, int64_t pts, AVRational tb);
|
||||
int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *sub);
|
||||
void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb);
|
||||
|
||||
/**
|
||||
* Set up fallback filtering parameters from a decoder context. They will only
|
||||
|
|
|
|||
|
|
@ -321,13 +321,8 @@ static int video_frame_process(InputStream *ist, AVFrame *frame)
|
|||
|
||||
static void sub2video_flush(InputStream *ist)
|
||||
{
|
||||
int i;
|
||||
int ret;
|
||||
|
||||
if (ist->sub2video.end_pts < INT64_MAX)
|
||||
sub2video_update(ist, INT64_MAX, NULL);
|
||||
for (i = 0; i < ist->nb_filters; i++) {
|
||||
ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
|
||||
for (int i = 0; i < ist->nb_filters; i++) {
|
||||
int ret = ifilter_sub2video(ist->filters[i], NULL);
|
||||
if (ret != AVERROR_EOF && ret < 0)
|
||||
av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -817,7 +817,6 @@ static void ist_free(InputStream **pist)
|
|||
|
||||
av_dict_free(&ist->decoder_opts);
|
||||
avsubtitle_free(&ist->prev_sub.subtitle);
|
||||
av_frame_free(&ist->sub2video.frame);
|
||||
av_freep(&ist->filters);
|
||||
av_freep(&ist->outputs);
|
||||
av_freep(&ist->hwaccel_device);
|
||||
|
|
|
|||
|
|
@ -107,6 +107,14 @@ typedef struct InputFilterPriv {
|
|||
struct {
|
||||
///< queue of AVSubtitle* before filter init
|
||||
AVFifo *queue;
|
||||
|
||||
AVFrame *frame;
|
||||
|
||||
int64_t last_pts;
|
||||
int64_t end_pts;
|
||||
|
||||
///< marks if sub2video_update should force an initialization
|
||||
unsigned int initialize;
|
||||
} sub2video;
|
||||
} InputFilterPriv;
|
||||
|
||||
|
|
@ -115,6 +123,111 @@ static InputFilterPriv *ifp_from_ifilter(InputFilter *ifilter)
|
|||
return (InputFilterPriv*)ifilter;
|
||||
}
|
||||
|
||||
static int sub2video_get_blank_frame(InputFilterPriv *ifp)
|
||||
{
|
||||
AVFrame *frame = ifp->sub2video.frame;
|
||||
int ret;
|
||||
|
||||
av_frame_unref(frame);
|
||||
|
||||
frame->width = ifp->width;
|
||||
frame->height = ifp->height;
|
||||
frame->format = ifp->format;
|
||||
|
||||
ret = av_frame_get_buffer(frame, 0);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
memset(frame->data[0], 0, frame->height * frame->linesize[0]);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
|
||||
AVSubtitleRect *r)
|
||||
{
|
||||
uint32_t *pal, *dst2;
|
||||
uint8_t *src, *src2;
|
||||
int x, y;
|
||||
|
||||
if (r->type != SUBTITLE_BITMAP) {
|
||||
av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
|
||||
return;
|
||||
}
|
||||
if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
|
||||
av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
|
||||
r->x, r->y, r->w, r->h, w, h
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
dst += r->y * dst_linesize + r->x * 4;
|
||||
src = r->data[0];
|
||||
pal = (uint32_t *)r->data[1];
|
||||
for (y = 0; y < r->h; y++) {
|
||||
dst2 = (uint32_t *)dst;
|
||||
src2 = src;
|
||||
for (x = 0; x < r->w; x++)
|
||||
*(dst2++) = pal[*(src2++)];
|
||||
dst += dst_linesize;
|
||||
src += r->linesize[0];
|
||||
}
|
||||
}
|
||||
|
||||
static void sub2video_push_ref(InputFilterPriv *ifp, int64_t pts)
|
||||
{
|
||||
AVFrame *frame = ifp->sub2video.frame;
|
||||
int ret;
|
||||
|
||||
av_assert1(frame->data[0]);
|
||||
ifp->sub2video.last_pts = frame->pts = pts;
|
||||
ret = av_buffersrc_add_frame_flags(ifp->ifilter.filter, frame,
|
||||
AV_BUFFERSRC_FLAG_KEEP_REF |
|
||||
AV_BUFFERSRC_FLAG_PUSH);
|
||||
if (ret != AVERROR_EOF && ret < 0)
|
||||
av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
|
||||
av_err2str(ret));
|
||||
}
|
||||
|
||||
static void sub2video_update(InputFilterPriv *ifp, int64_t heartbeat_pts,
|
||||
const AVSubtitle *sub)
|
||||
{
|
||||
AVFrame *frame = ifp->sub2video.frame;
|
||||
int8_t *dst;
|
||||
int dst_linesize;
|
||||
int num_rects, i;
|
||||
int64_t pts, end_pts;
|
||||
|
||||
if (sub) {
|
||||
pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
|
||||
AV_TIME_BASE_Q, ifp->time_base);
|
||||
end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL,
|
||||
AV_TIME_BASE_Q, ifp->time_base);
|
||||
num_rects = sub->num_rects;
|
||||
} else {
|
||||
/* If we are initializing the system, utilize current heartbeat
|
||||
PTS as the start time, and show until the following subpicture
|
||||
is received. Otherwise, utilize the previous subpicture's end time
|
||||
as the fall-back value. */
|
||||
pts = ifp->sub2video.initialize ?
|
||||
heartbeat_pts : ifp->sub2video.end_pts;
|
||||
end_pts = INT64_MAX;
|
||||
num_rects = 0;
|
||||
}
|
||||
if (sub2video_get_blank_frame(ifp) < 0) {
|
||||
av_log(NULL, AV_LOG_ERROR,
|
||||
"Impossible to get a blank canvas.\n");
|
||||
return;
|
||||
}
|
||||
dst = frame->data [0];
|
||||
dst_linesize = frame->linesize[0];
|
||||
for (i = 0; i < num_rects; i++)
|
||||
sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
|
||||
sub2video_push_ref(ifp, pts);
|
||||
ifp->sub2video.end_pts = end_pts;
|
||||
ifp->sub2video.initialize = 0;
|
||||
}
|
||||
|
||||
// FIXME: YUV420P etc. are actually supported with full color range,
|
||||
// yet the latter information isn't available here.
|
||||
static const enum AVPixelFormat *get_compliance_normal_pix_fmts(const AVCodec *codec, const enum AVPixelFormat default_formats[])
|
||||
|
|
@ -465,6 +578,12 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist)
|
|||
if (ret < 0)
|
||||
return ret;
|
||||
|
||||
if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE) {
|
||||
ifp->sub2video.frame = av_frame_alloc();
|
||||
if (!ifp->sub2video.frame)
|
||||
return AVERROR(ENOMEM);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -610,6 +729,7 @@ void fg_free(FilterGraph **pfg)
|
|||
avsubtitle_free(&sub);
|
||||
av_fifo_freep2(&ifp->sub2video.queue);
|
||||
}
|
||||
av_frame_free(&ifp->sub2video.frame);
|
||||
|
||||
av_channel_layout_uninit(&ifp->fallback.ch_layout);
|
||||
|
||||
|
|
@ -1108,20 +1228,15 @@ void check_filter_outputs(void)
|
|||
}
|
||||
}
|
||||
|
||||
static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
|
||||
static void sub2video_prepare(InputFilterPriv *ifp)
|
||||
{
|
||||
ist->sub2video.frame = av_frame_alloc();
|
||||
if (!ist->sub2video.frame)
|
||||
return AVERROR(ENOMEM);
|
||||
ist->sub2video.last_pts = INT64_MIN;
|
||||
ist->sub2video.end_pts = INT64_MIN;
|
||||
ifp->sub2video.last_pts = INT64_MIN;
|
||||
ifp->sub2video.end_pts = INT64_MIN;
|
||||
|
||||
/* sub2video structure has been (re-)initialized.
|
||||
Mark it as such so that the system will be
|
||||
initialized with the first received heartbeat. */
|
||||
ist->sub2video.initialize = 1;
|
||||
|
||||
return 0;
|
||||
ifp->sub2video.initialize = 1;
|
||||
}
|
||||
|
||||
static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
|
||||
|
|
@ -1156,11 +1271,8 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
|
|||
if (!fr.num)
|
||||
fr = ist->framerate_guessed;
|
||||
|
||||
if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
|
||||
ret = sub2video_prepare(ist, ifilter);
|
||||
if (ret < 0)
|
||||
goto fail;
|
||||
}
|
||||
if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE)
|
||||
sub2video_prepare(ifp);
|
||||
|
||||
ifp->time_base = ist->framerate.num ? av_inv_q(ist->framerate) :
|
||||
ist->st->time_base;
|
||||
|
|
@ -1466,12 +1578,13 @@ int configure_filtergraph(FilterGraph *fg)
|
|||
|
||||
/* process queued up subtitle packets */
|
||||
for (i = 0; i < fg->nb_inputs; i++) {
|
||||
InputFilterPriv *ifp = ifp_from_ifilter(fg->inputs[i]);
|
||||
InputStream *ist = ifp->ist;
|
||||
if (ifp->sub2video.queue && ist->sub2video.frame) {
|
||||
InputFilter *ifilter = fg->inputs[i];
|
||||
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
|
||||
|
||||
if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE && ifp->sub2video.queue) {
|
||||
AVSubtitle tmp;
|
||||
while (av_fifo_read(ifp->sub2video.queue, &tmp, 1) >= 0) {
|
||||
sub2video_update(ist, INT64_MIN, &tmp);
|
||||
sub2video_update(ifp, INT64_MIN, &tmp);
|
||||
avsubtitle_free(&tmp);
|
||||
}
|
||||
}
|
||||
|
|
@ -1620,15 +1733,47 @@ int reap_filters(int flush)
|
|||
return 0;
|
||||
}
|
||||
|
||||
void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb)
|
||||
{
|
||||
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
|
||||
int64_t pts2;
|
||||
|
||||
if (!ifilter->graph->graph)
|
||||
return;
|
||||
|
||||
/* subtitles seem to be usually muxed ahead of other streams;
|
||||
if not, subtracting a larger time here is necessary */
|
||||
pts2 = av_rescale_q(pts, tb, ifp->time_base) - 1;
|
||||
|
||||
/* do not send the heartbeat frame if the subtitle is already ahead */
|
||||
if (pts2 <= ifp->sub2video.last_pts)
|
||||
return;
|
||||
|
||||
if (pts2 >= ifp->sub2video.end_pts || ifp->sub2video.initialize)
|
||||
/* if we have hit the end of the current displayed subpicture,
|
||||
or if we need to initialize the system, update the
|
||||
overlayed subpicture and its start/end times */
|
||||
sub2video_update(ifp, pts2 + 1, NULL);
|
||||
|
||||
if (av_buffersrc_get_nb_failed_requests(ifilter->filter))
|
||||
sub2video_push_ref(ifp, pts2);
|
||||
}
|
||||
|
||||
int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *subtitle)
|
||||
{
|
||||
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
|
||||
InputStream *ist = ifp->ist;
|
||||
int ret;
|
||||
|
||||
if (ist->sub2video.frame) {
|
||||
sub2video_update(ist, INT64_MIN, subtitle);
|
||||
} else {
|
||||
if (ifilter->graph->graph) {
|
||||
if (!subtitle) {
|
||||
if (ifp->sub2video.end_pts < INT64_MAX)
|
||||
sub2video_update(ifp, INT64_MAX, NULL);
|
||||
|
||||
return av_buffersrc_add_frame(ifilter->filter, NULL);
|
||||
}
|
||||
|
||||
sub2video_update(ifp, INT64_MIN, subtitle);
|
||||
} else if (subtitle) {
|
||||
AVSubtitle sub;
|
||||
|
||||
if (!ifp->sub2video.queue)
|
||||
|
|
|
|||
Loading…
Reference in New Issue