fftools/ffmpeg: attach wallclock timing information to packets and frames

Will become useful in following commits.

fftools/ffmpeg.c

@@ -416,6 +416,9 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
 
         fd->dec.frame_num = UINT64_MAX;
         fd->dec.pts       = AV_NOPTS_VALUE;
+
+        for (unsigned i = 0; i < FF_ARRAY_ELEMS(fd->wallclock); i++)
+            fd->wallclock[i] = INT64_MIN;
     } else if (writable)
         return av_buffer_make_writable(dst);
 

fftools/ffmpeg.h

@@ -94,6 +94,17 @@ enum PacketOpaque {
     PKT_OPAQUE_FIX_SUB_DURATION,
 };
 
+enum LatencyProbe {
+    LATENCY_PROBE_DEMUX,
+    LATENCY_PROBE_DEC_PRE,
+    LATENCY_PROBE_DEC_POST,
+    LATENCY_PROBE_FILTER_PRE,
+    LATENCY_PROBE_FILTER_POST,
+    LATENCY_PROBE_ENC_PRE,
+    LATENCY_PROBE_ENC_POST,
+    LATENCY_PROBE_NB,
+};
+
 typedef struct HWDevice {
     const char *name;
     enum AVHWDeviceType type;
@@ -631,6 +642,8 @@ typedef struct FrameData {
     AVRational frame_rate_filter;
 
     int        bits_per_raw_sample;
+
+    int64_t wallclock[LATENCY_PROBE_NB];
 } FrameData;
 
 extern InputFile   **input_files;

fftools/ffmpeg_dec.c

@@ -22,6 +22,7 @@
 #include "libavutil/log.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/pixfmt.h"
+#include "libavutil/time.h"
 #include "libavutil/timestamp.h"
 
 #include "libavcodec/avcodec.h"
@@ -475,6 +476,13 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame)
         pkt->dts = AV_NOPTS_VALUE;
     }
 
+    if (pkt) {
+        FrameData *fd = packet_data(pkt);
+        if (!fd)
+            return AVERROR(ENOMEM);
+        fd->wallclock[LATENCY_PROBE_DEC_PRE] = av_gettime_relative();
+    }
+
     ret = avcodec_send_packet(dec, pkt);
     if (ret < 0 && !(ret == AVERROR_EOF && !pkt)) {
         // In particular, we don't expect AVERROR(EAGAIN), because we read all
@@ -528,8 +536,6 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame)
                 return AVERROR_INVALIDDATA;
         }
 
-
-        av_assert0(!frame->opaque_ref);
         fd      = frame_data(frame);
         if (!fd) {
             av_frame_unref(frame);
@@ -540,6 +546,8 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame)
         fd->dec.frame_num           = dec->frame_num - 1;
         fd->bits_per_raw_sample     = dec->bits_per_raw_sample;
 
+        fd->wallclock[LATENCY_PROBE_DEC_POST] = av_gettime_relative();
+
         frame->time_base = dec->pkt_timebase;
 
         if (dec->codec_type == AVMEDIA_TYPE_AUDIO) {
@@ -932,6 +940,8 @@ int dec_open(InputStream *ist, Scheduler *sch, unsigned sch_idx)
     if (ist->st->disposition & AV_DISPOSITION_ATTACHED_PIC)
         av_dict_set(&ist->decoder_opts, "threads", "1", 0);
 
+    av_dict_set(&ist->decoder_opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
+
     ret = hw_device_setup_for_decode(ist);
     if (ret < 0) {
         av_log(ist, AV_LOG_ERROR,

fftools/ffmpeg_demux.c

@@ -271,7 +271,7 @@ static void ts_discontinuity_process(Demuxer *d, InputStream *ist,
         ts_discontinuity_detect(d, ist, pkt);
 }
 
-static int ist_dts_update(DemuxStream *ds, AVPacket *pkt)
+static int ist_dts_update(DemuxStream *ds, AVPacket *pkt, FrameData *fd)
 {
     InputStream *ist = &ds->ist;
     const AVCodecParameters *par = ist->par;
@@ -326,21 +326,12 @@ static int ist_dts_update(DemuxStream *ds, AVPacket *pkt)
         break;
     }
 
-    av_assert0(!pkt->opaque_ref);
-    if (ds->streamcopy_needed) {
-        FrameData *fd;
-
-        fd = packet_data(pkt);
-        if (!fd)
-            return AVERROR(ENOMEM);
-
-        fd->dts_est = ds->dts;
-    }
+    fd->dts_est = ds->dts;
 
     return 0;
 }
 
-static int ts_fixup(Demuxer *d, AVPacket *pkt)
+static int ts_fixup(Demuxer *d, AVPacket *pkt, FrameData *fd)
 {
     InputFile *ifile = &d->f;
     InputStream *ist = ifile->streams[pkt->stream_index];
@@ -424,7 +415,7 @@ static int ts_fixup(Demuxer *d, AVPacket *pkt)
     ts_discontinuity_process(d, ist, pkt);
 
     // update estimated/predicted dts
-    ret = ist_dts_update(ds, pkt);
+    ret = ist_dts_update(ds, pkt, fd);
     if (ret < 0)
         return ret;
 
@@ -436,9 +427,14 @@ static int input_packet_process(Demuxer *d, AVPacket *pkt, unsigned *send_flags)
     InputFile     *f = &d->f;
     InputStream *ist = f->streams[pkt->stream_index];
     DemuxStream  *ds = ds_from_ist(ist);
+    FrameData *fd;
     int ret = 0;
 
-    ret = ts_fixup(d, pkt);
+    fd = packet_data(pkt);
+    if (!fd)
+        return AVERROR(ENOMEM);
+
+    ret = ts_fixup(d, pkt, fd);
     if (ret < 0)
         return ret;
 
@@ -455,6 +451,8 @@ static int input_packet_process(Demuxer *d, AVPacket *pkt, unsigned *send_flags)
     ds->data_size += pkt->size;
     ds->nb_packets++;
 
+    fd->wallclock[LATENCY_PROBE_DEMUX] = av_gettime_relative();
+
     if (debug_ts) {
         av_log(NULL, AV_LOG_INFO, "demuxer+ffmpeg -> ist_index:%d:%d type:%s pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s duration:%s duration_time:%s off:%s off_time:%s\n",
                f->index, pkt->stream_index,

fftools/ffmpeg_enc.c

@@ -34,6 +34,7 @@
 #include "libavutil/log.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/rational.h"
+#include "libavutil/time.h"
 #include "libavutil/timestamp.h"
 
 #include "libavcodec/avcodec.h"
@@ -615,6 +616,14 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
     int ret;
 
     if (frame) {
+        FrameData *fd = frame_data(frame);
+
+        fd = frame_data(frame);
+        if (!fd)
+            return AVERROR(ENOMEM);
+
+        fd->wallclock[LATENCY_PROBE_ENC_PRE] = av_gettime_relative();
+
         if (ost->enc_stats_pre.io)
             enc_stats_write(ost, &ost->enc_stats_pre, frame, NULL,
                             ost->frames_encoded);
@@ -644,6 +653,8 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
     }
 
     while (1) {
+        FrameData *fd;
+
         av_packet_unref(pkt);
 
         ret = avcodec_receive_packet(enc, pkt);
@@ -665,6 +676,11 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
             return ret;
         }
 
+        fd = packet_data(pkt);
+        if (!fd)
+            return AVERROR(ENOMEM);
+        fd->wallclock[LATENCY_PROBE_ENC_POST] = av_gettime_relative();
+
         if (enc->codec_type == AVMEDIA_TYPE_VIDEO) {
             ret = update_video_stats(ost, pkt, !!vstats_filename);
             if (ret < 0)

fftools/ffmpeg_filter.c

@@ -36,6 +36,7 @@
 #include "libavutil/pixfmt.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/samplefmt.h"
+#include "libavutil/time.h"
 #include "libavutil/timestamp.h"
 
 // FIXME private header, used for mid_pred()
@@ -2364,6 +2365,8 @@ static int fg_output_step(OutputFilterPriv *ofp, FilterGraphThread *fgt,
         return AVERROR(ENOMEM);
     }
 
+    fd->wallclock[LATENCY_PROBE_FILTER_POST] = av_gettime_relative();
+
     // only use bits_per_raw_sample passed through from the decoder
     // if the filtergraph did not touch the frame data
     if (!fgp->is_meta)
@@ -2576,6 +2579,7 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt,
                       InputFilter *ifilter, AVFrame *frame)
 {
     InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+    FrameData       *fd;
     AVFrameSideData *sd;
     int need_reinit, ret;
 
@@ -2651,6 +2655,11 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt,
     )
 #endif
 
+    fd = frame_data(frame);
+    if (!fd)
+        return AVERROR(ENOMEM);
+    fd->wallclock[LATENCY_PROBE_FILTER_PRE] = av_gettime_relative();
+
     ret = av_buffersrc_add_frame_flags(ifp->filter, frame,
                                        AV_BUFFERSRC_FLAG_PUSH);
     if (ret < 0) {