diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index b273f3e477..b407b6f907 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -3194,9 +3194,6 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame)
     if (ost->bitexact)
         enc_ctx->flags |= AV_CODEC_FLAG_BITEXACT;
 
-    if (ost->sq_idx_encode >= 0)
-        sq_set_tb(of->sq_encode, ost->sq_idx_encode, enc_ctx->time_base);
-
     ost->mux_timebase = enc_ctx->time_base;
 
     return 0;
diff --git a/fftools/ffmpeg_mux.c b/fftools/ffmpeg_mux.c
index cf58051949..1937bc2aa7 100644
--- a/fftools/ffmpeg_mux.c
+++ b/fftools/ffmpeg_mux.c
@@ -586,9 +586,6 @@ int of_stream_init(OutputFile *of, OutputStream *ost)
     MuxStream *ms = ms_from_ost(ost);
     int ret;
 
-    if (ost->sq_idx_mux >= 0)
-        sq_set_tb(mux->sq_mux, ost->sq_idx_mux, ost->mux_timebase);
-
     /* initialize bitstream filters for the output stream
      * needs to be done here, because the codec id for streamcopy is not
      * known until now */
diff --git a/fftools/sync_queue.c b/fftools/sync_queue.c
index c2b23ee4f5..7c348af300 100644
--- a/fftools/sync_queue.c
+++ b/fftools/sync_queue.c
@@ -84,6 +84,26 @@ static int frame_null(const SyncQueue *sq, SyncQueueFrame frame)
     return (sq->type == SYNC_QUEUE_PACKETS) ? (frame.p == NULL) : (frame.f == NULL);
 }
 
+static void tb_update(const SyncQueue *sq, SyncQueueStream *st,
+                      const SyncQueueFrame frame)
+{
+    AVRational tb = (sq->type == SYNC_QUEUE_PACKETS) ?
+                    frame.p->time_base : frame.f->time_base;
+
+    av_assert0(tb.num > 0 && tb.den > 0);
+
+    if (tb.num == st->tb.num && tb.den == st->tb.den)
+        return;
+
+    // timebase should not change after the first frame
+    av_assert0(!av_fifo_can_read(st->fifo));
+
+    if (st->head_ts != AV_NOPTS_VALUE)
+        st->head_ts = av_rescale_q(st->head_ts, st->tb, tb);
+
+    st->tb = tb;
+}
+
 static void finish_stream(SyncQueue *sq, unsigned int stream_idx)
 {
     SyncQueueStream *st = &sq->streams[stream_idx];
@@ -241,8 +261,6 @@ int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame)
     av_assert0(stream_idx < sq->nb_streams);
     st = &sq->streams[stream_idx];
 
-    av_assert0(st->tb.num > 0 && st->tb.den > 0);
-
     if (frame_null(sq, frame)) {
         finish_stream(sq, stream_idx);
         return 0;
@@ -250,6 +268,8 @@ int sq_send(SyncQueue *sq, unsigned int stream_idx, SyncQueueFrame frame)
     if (st->finished)
         return AVERROR_EOF;
 
+    tb_update(sq, st, frame);
+
     ret = objpool_get(sq->pool, (void**)&dst);
     if (ret < 0)
         return ret;
@@ -375,21 +395,6 @@ int sq_add_stream(SyncQueue *sq, int limiting)
     return sq->nb_streams++;
 }
 
-void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb)
-{
-    SyncQueueStream *st;
-
-    av_assert0(stream_idx < sq->nb_streams);
-    st = &sq->streams[stream_idx];
-
-    av_assert0(!av_fifo_can_read(st->fifo));
-
-    if (st->head_ts != AV_NOPTS_VALUE)
-        st->head_ts = av_rescale_q(st->head_ts, st->tb, tb);
-
-    st->tb = tb;
-}
-
 void sq_limit_frames(SyncQueue *sq, unsigned int stream_idx, uint64_t frames)
 {
     SyncQueueStream *st;
diff --git a/fftools/sync_queue.h b/fftools/sync_queue.h
index 3f823ff0d9..17d282c38c 100644
--- a/fftools/sync_queue.h
+++ b/fftools/sync_queue.h
@@ -59,12 +59,6 @@ void       sq_free(SyncQueue **sq);
  */
 int sq_add_stream(SyncQueue *sq, int limiting);
 
-/**
- * Set the timebase for the stream with index stream_idx. Should be called
- * before sending any frames for this stream.
- */
-void sq_set_tb(SyncQueue *sq, unsigned int stream_idx, AVRational tb);
-
 /**
  * Limit the number of output frames for stream with index stream_idx
  * to max_frames.