diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 9cf94f2a22..40e97ad486 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -1498,14 +1498,6 @@ static int transcode_init(void)
 {
     int ret = 0;
 
-    /* init framerate emulation */
-    for (int i = 0; i < nb_input_files; i++) {
-        InputFile *ifile = input_files[i];
-        if (ifile->readrate)
-            for (int j = 0; j < ifile->nb_streams; j++)
-                ifile->streams[j]->start = av_gettime_relative();
-    }
-
     /* discard unused programs */
     for (int i = 0; i < nb_input_files; i++) {
         InputFile *ifile = input_files[i];
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 7eb6301c74..d1af94590d 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -362,8 +362,6 @@ typedef struct InputStream {
 
     AVRational framerate_guessed;
 
-    int64_t       start;     /* time when read started */
-
     // pts/estimated duration of the last decoded frame
     // * in decoder timebase for video,
     // * in last_frame_tb (may change during decoding) for audio
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index ccf1a70d5e..aff80b1cd3 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -88,6 +88,8 @@ typedef struct Demuxer {
     // name used for logging
     char log_name[32];
 
+    int64_t wallclock_start;
+
     /* number of times input stream should be looped */
     int loop;
     /* actual duration of the longest stream in a file at the moment when
@@ -517,7 +519,7 @@ static void readrate_sleep(Demuxer *d)
         int64_t stream_ts_offset, pts, now;
         stream_ts_offset = FFMAX(ds->first_dts != AV_NOPTS_VALUE ? ds->first_dts : 0, file_start);
         pts = av_rescale(ds->dts, 1000000, AV_TIME_BASE);
-        now = (av_gettime_relative() - ist->start) * f->readrate + stream_ts_offset;
+        now = (av_gettime_relative() - d->wallclock_start) * f->readrate + stream_ts_offset;
         if (pts - burst_until > now)
             av_usleep(pts - burst_until - now);
     }
@@ -546,6 +548,8 @@ static void *input_thread(void *arg)
 
     thread_set_name(f);
 
+    d->wallclock_start = av_gettime_relative();
+
     while (1) {
         DemuxMsg msg = { NULL };