diff --git a/fftools/ffmpeg_sched.c b/fftools/ffmpeg_sched.c
index 318ec44b73..6a58661a5c 100644
--- a/fftools/ffmpeg_sched.c
+++ b/fftools/ffmpeg_sched.c
@@ -285,6 +285,7 @@ struct Scheduler {
     pthread_mutex_t     mux_ready_lock;
 
     unsigned         nb_mux_done;
+    unsigned            task_failed;
     pthread_mutex_t     finish_lock;
     pthread_cond_t      finish_cond;
 
@@ -306,7 +307,6 @@ struct Scheduler {
 
     enum SchedulerState state;
     atomic_int          terminate;
-    atomic_int          task_failed;
 
     pthread_mutex_t     schedule_lock;
 
@@ -1664,7 +1664,7 @@ fail:
 
 int sch_wait(Scheduler *sch, uint64_t timeout_us, int64_t *transcode_ts)
 {
-    int ret, err;
+    int ret;
 
     // convert delay to absolute timestamp
     timeout_us += av_gettime();
@@ -1677,16 +1677,14 @@ int sch_wait(Scheduler *sch, uint64_t timeout_us, int64_t *transcode_ts)
         pthread_cond_timedwait(&sch->finish_cond, &sch->finish_lock, &tv);
     }
 
-    ret = sch->nb_mux_done == sch->nb_mux;
+    // abort transcoding if any task failed
+    ret = sch->nb_mux_done == sch->nb_mux || sch->task_failed;
 
     pthread_mutex_unlock(&sch->finish_lock);
 
     *transcode_ts = atomic_load(&sch->last_dts);
 
-    // abort transcoding if any task failed
-    err = atomic_load(&sch->task_failed);
-
-    return ret || err;
+    return ret;
 }
 
 static int enc_open(Scheduler *sch, SchEnc *enc, const AVFrame *frame)
@@ -2552,8 +2550,12 @@ static void *task_wrapper(void *arg)
     // EOF is considered normal termination
     if (ret == AVERROR_EOF)
         ret = 0;
-    if (ret < 0)
-        atomic_store(&sch->task_failed, 1);
+    if (ret < 0) {
+        pthread_mutex_lock(&sch->finish_lock);
+        sch->task_failed = 1;
+        pthread_cond_signal(&sch->finish_cond);
+        pthread_mutex_unlock(&sch->finish_lock);
+    }
 
     av_log(task->func_arg, ret < 0 ? AV_LOG_ERROR : AV_LOG_VERBOSE,
            "Terminating thread with return code %d (%s)\n", ret,