diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c index e9e60407d2..36b4becaf2 100644 --- a/fftools/ffmpeg.c +++ b/fftools/ffmpeg.c @@ -147,139 +147,20 @@ static int restore_tty; This is a temporary solution until libavfilter gets real subtitles support. */ -static int sub2video_get_blank_frame(InputStream *ist) -{ - int ret; - AVFrame *frame = ist->sub2video.frame; - - av_frame_unref(frame); - frame->width = ist->sub2video.w; - frame->height = ist->sub2video.h; - frame->format = AV_PIX_FMT_RGB32; - if ((ret = av_frame_get_buffer(frame, 0)) < 0) - return ret; - memset(frame->data[0], 0, frame->height * frame->linesize[0]); - return 0; -} - -static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h, - AVSubtitleRect *r) -{ - uint32_t *pal, *dst2; - uint8_t *src, *src2; - int x, y; - - if (r->type != SUBTITLE_BITMAP) { - av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n"); - return; - } - if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) { - av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n", - r->x, r->y, r->w, r->h, w, h - ); - return; - } - - dst += r->y * dst_linesize + r->x * 4; - src = r->data[0]; - pal = (uint32_t *)r->data[1]; - for (y = 0; y < r->h; y++) { - dst2 = (uint32_t *)dst; - src2 = src; - for (x = 0; x < r->w; x++) - *(dst2++) = pal[*(src2++)]; - dst += dst_linesize; - src += r->linesize[0]; - } -} - -static void sub2video_push_ref(InputStream *ist, int64_t pts) -{ - AVFrame *frame = ist->sub2video.frame; - int i; - int ret; - - av_assert1(frame->data[0]); - ist->sub2video.last_pts = frame->pts = pts; - for (i = 0; i < ist->nb_filters; i++) { - ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame, - AV_BUFFERSRC_FLAG_KEEP_REF | - AV_BUFFERSRC_FLAG_PUSH); - if (ret != AVERROR_EOF && ret < 0) - av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n", - av_err2str(ret)); - } -} - -void sub2video_update(InputStream *ist, int64_t heartbeat_pts, - const AVSubtitle *sub) -{ - AVFrame *frame = ist->sub2video.frame; - int8_t *dst; - int dst_linesize; - int num_rects, i; - int64_t pts, end_pts; - - if (!frame) - return; - if (sub) { - pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL, - AV_TIME_BASE_Q, ist->st->time_base); - end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL, - AV_TIME_BASE_Q, ist->st->time_base); - num_rects = sub->num_rects; - } else { - /* If we are initializing the system, utilize current heartbeat - PTS as the start time, and show until the following subpicture - is received. Otherwise, utilize the previous subpicture's end time - as the fall-back value. */ - pts = ist->sub2video.initialize ? - heartbeat_pts : ist->sub2video.end_pts; - end_pts = INT64_MAX; - num_rects = 0; - } - if (sub2video_get_blank_frame(ist) < 0) { - av_log(NULL, AV_LOG_ERROR, - "Impossible to get a blank canvas.\n"); - return; - } - dst = frame->data [0]; - dst_linesize = frame->linesize[0]; - for (i = 0; i < num_rects; i++) - sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]); - sub2video_push_ref(ist, pts); - ist->sub2video.end_pts = end_pts; - ist->sub2video.initialize = 0; -} - static void sub2video_heartbeat(InputFile *infile, int64_t pts, AVRational tb) { - int i, j, nb_reqs; - int64_t pts2; - /* When a frame is read from a file, examine all sub2video streams in the same file and send the sub2video frame again. Otherwise, decoded video frames could be accumulating in the filter graph while a filter (possibly overlay) is desperately waiting for a subtitle frame. */ - for (i = 0; i < infile->nb_streams; i++) { - InputStream *ist2 = infile->streams[i]; - if (!ist2->sub2video.frame) + for (int i = 0; i < infile->nb_streams; i++) { + InputStream *ist = infile->streams[i]; + + if (ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE) continue; - /* subtitles seem to be usually muxed ahead of other streams; - if not, subtracting a larger time here is necessary */ - pts2 = av_rescale_q(pts, tb, ist2->st->time_base) - 1; - /* do not send the heartbeat frame if the subtitle is already ahead */ - if (pts2 <= ist2->sub2video.last_pts) - continue; - if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize) - /* if we have hit the end of the current displayed subpicture, - or if we need to initialize the system, update the - overlayed subpicture and its start/end times */ - sub2video_update(ist2, pts2 + 1, NULL); - for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++) - nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter); - if (nb_reqs) - sub2video_push_ref(ist2, pts2); + + for (int j = 0; j < ist->nb_filters; j++) + ifilter_sub2video_heartbeat(ist->filters[j], pts, tb); } } diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h index d4aff5cc7c..75695d3fb5 100644 --- a/fftools/ffmpeg.h +++ b/fftools/ffmpeg.h @@ -372,11 +372,7 @@ typedef struct InputStream { } prev_sub; struct sub2video { - int64_t last_pts; - int64_t end_pts; - AVFrame *frame; int w, h; - unsigned int initialize; ///< marks if sub2video_update should force an initialization } sub2video; /* decoded data from this stream goes into all those filters @@ -741,13 +737,12 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost, char *graph_desc); int init_complex_filtergraph(FilterGraph *fg); -void sub2video_update(InputStream *ist, int64_t heartbeat_pts, - const AVSubtitle *sub); int copy_av_subtitle(AVSubtitle *dst, const AVSubtitle *src); int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame, int keep_reference); int ifilter_send_eof(InputFilter *ifilter, int64_t pts, AVRational tb); int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *sub); +void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb); /** * Set up fallback filtering parameters from a decoder context. They will only diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c index a7acaf67c2..30959c64b7 100644 --- a/fftools/ffmpeg_dec.c +++ b/fftools/ffmpeg_dec.c @@ -321,13 +321,8 @@ static int video_frame_process(InputStream *ist, AVFrame *frame) static void sub2video_flush(InputStream *ist) { - int i; - int ret; - - if (ist->sub2video.end_pts < INT64_MAX) - sub2video_update(ist, INT64_MAX, NULL); - for (i = 0; i < ist->nb_filters; i++) { - ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL); + for (int i = 0; i < ist->nb_filters; i++) { + int ret = ifilter_sub2video(ist->filters[i], NULL); if (ret != AVERROR_EOF && ret < 0) av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n"); } diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c index 817ccbbedc..5c15b8bad3 100644 --- a/fftools/ffmpeg_demux.c +++ b/fftools/ffmpeg_demux.c @@ -817,7 +817,6 @@ static void ist_free(InputStream **pist) av_dict_free(&ist->decoder_opts); avsubtitle_free(&ist->prev_sub.subtitle); - av_frame_free(&ist->sub2video.frame); av_freep(&ist->filters); av_freep(&ist->outputs); av_freep(&ist->hwaccel_device); diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c index 4b5ccf50bd..8f839cc273 100644 --- a/fftools/ffmpeg_filter.c +++ b/fftools/ffmpeg_filter.c @@ -107,6 +107,14 @@ typedef struct InputFilterPriv { struct { ///< queue of AVSubtitle* before filter init AVFifo *queue; + + AVFrame *frame; + + int64_t last_pts; + int64_t end_pts; + + ///< marks if sub2video_update should force an initialization + unsigned int initialize; } sub2video; } InputFilterPriv; @@ -115,6 +123,111 @@ static InputFilterPriv *ifp_from_ifilter(InputFilter *ifilter) return (InputFilterPriv*)ifilter; } +static int sub2video_get_blank_frame(InputFilterPriv *ifp) +{ + AVFrame *frame = ifp->sub2video.frame; + int ret; + + av_frame_unref(frame); + + frame->width = ifp->width; + frame->height = ifp->height; + frame->format = ifp->format; + + ret = av_frame_get_buffer(frame, 0); + if (ret < 0) + return ret; + + memset(frame->data[0], 0, frame->height * frame->linesize[0]); + + return 0; +} + +static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h, + AVSubtitleRect *r) +{ + uint32_t *pal, *dst2; + uint8_t *src, *src2; + int x, y; + + if (r->type != SUBTITLE_BITMAP) { + av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n"); + return; + } + if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) { + av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n", + r->x, r->y, r->w, r->h, w, h + ); + return; + } + + dst += r->y * dst_linesize + r->x * 4; + src = r->data[0]; + pal = (uint32_t *)r->data[1]; + for (y = 0; y < r->h; y++) { + dst2 = (uint32_t *)dst; + src2 = src; + for (x = 0; x < r->w; x++) + *(dst2++) = pal[*(src2++)]; + dst += dst_linesize; + src += r->linesize[0]; + } +} + +static void sub2video_push_ref(InputFilterPriv *ifp, int64_t pts) +{ + AVFrame *frame = ifp->sub2video.frame; + int ret; + + av_assert1(frame->data[0]); + ifp->sub2video.last_pts = frame->pts = pts; + ret = av_buffersrc_add_frame_flags(ifp->ifilter.filter, frame, + AV_BUFFERSRC_FLAG_KEEP_REF | + AV_BUFFERSRC_FLAG_PUSH); + if (ret != AVERROR_EOF && ret < 0) + av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n", + av_err2str(ret)); +} + +static void sub2video_update(InputFilterPriv *ifp, int64_t heartbeat_pts, + const AVSubtitle *sub) +{ + AVFrame *frame = ifp->sub2video.frame; + int8_t *dst; + int dst_linesize; + int num_rects, i; + int64_t pts, end_pts; + + if (sub) { + pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL, + AV_TIME_BASE_Q, ifp->time_base); + end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL, + AV_TIME_BASE_Q, ifp->time_base); + num_rects = sub->num_rects; + } else { + /* If we are initializing the system, utilize current heartbeat + PTS as the start time, and show until the following subpicture + is received. Otherwise, utilize the previous subpicture's end time + as the fall-back value. */ + pts = ifp->sub2video.initialize ? + heartbeat_pts : ifp->sub2video.end_pts; + end_pts = INT64_MAX; + num_rects = 0; + } + if (sub2video_get_blank_frame(ifp) < 0) { + av_log(NULL, AV_LOG_ERROR, + "Impossible to get a blank canvas.\n"); + return; + } + dst = frame->data [0]; + dst_linesize = frame->linesize[0]; + for (i = 0; i < num_rects; i++) + sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]); + sub2video_push_ref(ifp, pts); + ifp->sub2video.end_pts = end_pts; + ifp->sub2video.initialize = 0; +} + // FIXME: YUV420P etc. are actually supported with full color range, // yet the latter information isn't available here. static const enum AVPixelFormat *get_compliance_normal_pix_fmts(const AVCodec *codec, const enum AVPixelFormat default_formats[]) @@ -465,6 +578,12 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist) if (ret < 0) return ret; + if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE) { + ifp->sub2video.frame = av_frame_alloc(); + if (!ifp->sub2video.frame) + return AVERROR(ENOMEM); + } + return 0; } @@ -610,6 +729,7 @@ void fg_free(FilterGraph **pfg) avsubtitle_free(&sub); av_fifo_freep2(&ifp->sub2video.queue); } + av_frame_free(&ifp->sub2video.frame); av_channel_layout_uninit(&ifp->fallback.ch_layout); @@ -1108,20 +1228,15 @@ void check_filter_outputs(void) } } -static int sub2video_prepare(InputStream *ist, InputFilter *ifilter) +static void sub2video_prepare(InputFilterPriv *ifp) { - ist->sub2video.frame = av_frame_alloc(); - if (!ist->sub2video.frame) - return AVERROR(ENOMEM); - ist->sub2video.last_pts = INT64_MIN; - ist->sub2video.end_pts = INT64_MIN; + ifp->sub2video.last_pts = INT64_MIN; + ifp->sub2video.end_pts = INT64_MIN; /* sub2video structure has been (re-)initialized. Mark it as such so that the system will be initialized with the first received heartbeat. */ - ist->sub2video.initialize = 1; - - return 0; + ifp->sub2video.initialize = 1; } static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter, @@ -1156,11 +1271,8 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter, if (!fr.num) fr = ist->framerate_guessed; - if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) { - ret = sub2video_prepare(ist, ifilter); - if (ret < 0) - goto fail; - } + if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE) + sub2video_prepare(ifp); ifp->time_base = ist->framerate.num ? av_inv_q(ist->framerate) : ist->st->time_base; @@ -1466,12 +1578,13 @@ int configure_filtergraph(FilterGraph *fg) /* process queued up subtitle packets */ for (i = 0; i < fg->nb_inputs; i++) { - InputFilterPriv *ifp = ifp_from_ifilter(fg->inputs[i]); - InputStream *ist = ifp->ist; - if (ifp->sub2video.queue && ist->sub2video.frame) { + InputFilter *ifilter = fg->inputs[i]; + InputFilterPriv *ifp = ifp_from_ifilter(ifilter); + + if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE && ifp->sub2video.queue) { AVSubtitle tmp; while (av_fifo_read(ifp->sub2video.queue, &tmp, 1) >= 0) { - sub2video_update(ist, INT64_MIN, &tmp); + sub2video_update(ifp, INT64_MIN, &tmp); avsubtitle_free(&tmp); } } @@ -1620,15 +1733,47 @@ int reap_filters(int flush) return 0; } +void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb) +{ + InputFilterPriv *ifp = ifp_from_ifilter(ifilter); + int64_t pts2; + + if (!ifilter->graph->graph) + return; + + /* subtitles seem to be usually muxed ahead of other streams; + if not, subtracting a larger time here is necessary */ + pts2 = av_rescale_q(pts, tb, ifp->time_base) - 1; + + /* do not send the heartbeat frame if the subtitle is already ahead */ + if (pts2 <= ifp->sub2video.last_pts) + return; + + if (pts2 >= ifp->sub2video.end_pts || ifp->sub2video.initialize) + /* if we have hit the end of the current displayed subpicture, + or if we need to initialize the system, update the + overlayed subpicture and its start/end times */ + sub2video_update(ifp, pts2 + 1, NULL); + + if (av_buffersrc_get_nb_failed_requests(ifilter->filter)) + sub2video_push_ref(ifp, pts2); +} + int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *subtitle) { InputFilterPriv *ifp = ifp_from_ifilter(ifilter); - InputStream *ist = ifp->ist; int ret; - if (ist->sub2video.frame) { - sub2video_update(ist, INT64_MIN, subtitle); - } else { + if (ifilter->graph->graph) { + if (!subtitle) { + if (ifp->sub2video.end_pts < INT64_MAX) + sub2video_update(ifp, INT64_MAX, NULL); + + return av_buffersrc_add_frame(ifilter->filter, NULL); + } + + sub2video_update(ifp, INT64_MIN, subtitle); + } else if (subtitle) { AVSubtitle sub; if (!ifp->sub2video.queue)