From 0c3e3fd1b478e14692f3b02bb8bf42262ee18af0 Mon Sep 17 00:00:00 2001
From: James Almer <jamrial@gmail.com>
Date: Tue, 2 Aug 2022 08:28:48 -0300
Subject: [PATCH] avfilter/framesync: add a new option to set how to sync
 streams based on secondary input timestamps

Include two values for it, a default one that sets/keeps the current behavior,
where the frame event generated by the primary input will have a timestamp
equal or higher than frames in secondary input, plus a new one where the
secondary input frame will be that with the absolute closest timestamp to that
of the frame event one.

Addresses ticket #9689, where the new optional behavior produces better frame
syncronization.

Reviewed-by: Nicolas George <george@nsup.org>
Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavfilter/framesync.c | 19 +++++++++++++++++++
 libavfilter/framesync.h | 23 +++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/libavfilter/framesync.c b/libavfilter/framesync.c
index 7510550d8e..ee91e4cf68 100644
--- a/libavfilter/framesync.c
+++ b/libavfilter/framesync.c
@@ -42,6 +42,13 @@ static const AVOption framesync_options[] = {
         { "pass",   "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS },   .flags = FLAGS, "eof_action" },
     { "shortest", "force termination when the shortest input terminates", OFFSET(opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
     { "repeatlast", "extend last frame of secondary streams beyond EOF", OFFSET(opt_repeatlast), AV_OPT_TYPE_BOOL, { .i64 = 1 }, 0, 1, FLAGS },
+    { "ts_sync_mode", "How strictly to sync streams based on secondary input timestamps",
+        OFFSET(opt_ts_sync_mode), AV_OPT_TYPE_INT, { .i64 = TS_DEFAULT },
+        TS_DEFAULT, TS_NEAREST, .flags = FLAGS, "ts_sync_mode" },
+        { "default", "Frame from secondary input with the nearest lower or equal timestamp to the primary input frame",
+            0, AV_OPT_TYPE_CONST, { .i64 = TS_DEFAULT }, .flags = FLAGS, "ts_sync_mode" },
+        { "nearest", "Frame from secondary input with the absolute nearest timestamp to the primary input frame",
+            0, AV_OPT_TYPE_CONST, { .i64 = TS_NEAREST }, .flags = FLAGS, "ts_sync_mode" },
     { NULL }
 };
 static const AVClass framesync_class = {
@@ -110,6 +117,14 @@ static void framesync_sync_level_update(FFFrameSync *fs)
     av_assert0(level <= fs->sync_level);
     if (level < fs->sync_level)
         av_log(fs, AV_LOG_VERBOSE, "Sync level %u\n", level);
+    if (fs->opt_ts_sync_mode > TS_DEFAULT) {
+        for (i = 0; i < fs->nb_in; i++) {
+            if (fs->in[i].sync < level)
+                fs->in[i].ts_mode = fs->opt_ts_sync_mode;
+            else
+                fs->in[i].ts_mode = TS_DEFAULT;
+        }
+    }
     if (level)
         fs->sync_level = level;
     else
@@ -187,6 +202,10 @@ static int framesync_advance(FFFrameSync *fs)
         }
         for (i = 0; i < fs->nb_in; i++) {
             if (fs->in[i].pts_next == pts ||
+                (fs->in[i].ts_mode == TS_NEAREST &&
+                 fs->in[i].have_next &&
+                 fs->in[i].pts_next != INT64_MAX && fs->in[i].pts != AV_NOPTS_VALUE &&
+                 fs->in[i].pts_next - pts < pts - fs->in[i].pts) ||
                 (fs->in[i].before == EXT_INFINITY &&
                  fs->in[i].state == STATE_BOF)) {
                 av_frame_free(&fs->in[i].frame);
diff --git a/libavfilter/framesync.h b/libavfilter/framesync.h
index a246d2d1e5..233f50a0eb 100644
--- a/libavfilter/framesync.h
+++ b/libavfilter/framesync.h
@@ -75,6 +75,27 @@ enum FFFrameSyncExtMode {
     EXT_INFINITY,
 };
 
+/**
+ * Timestamp syncronization mode
+ *
+ * Describe how the frames of a stream are syncronized based on timestamp
+ * distance.
+ */
+enum FFFrameTSSyncMode {
+
+    /**
+     * Sync to frames from secondary input with the nearest, lower or equal
+     * timestamp to the frame event one.
+     */
+    TS_DEFAULT,
+
+    /**
+     * Sync to frames from secondary input with the absolute nearest timestamp
+     * to the frame event one.
+     */
+    TS_NEAREST,
+};
+
 /**
  * Input stream structure
  */
@@ -138,6 +159,7 @@ typedef struct FFFrameSyncIn {
      */
     unsigned sync;
 
+    enum FFFrameTSSyncMode ts_mode;
 } FFFrameSyncIn;
 
 /**
@@ -205,6 +227,7 @@ typedef struct FFFrameSync {
     int opt_repeatlast;
     int opt_shortest;
     int opt_eof_action;
+    int opt_ts_sync_mode;
 
 } FFFrameSync;