diff --git a/audio/out/ao.c b/audio/out/ao.c
index 8f6fc8ea3c..8fd24c2439 100644
--- a/audio/out/ao.c
+++ b/audio/out/ao.c
@@ -422,7 +422,7 @@ int ao_query_and_reset_events(struct ao *ao, int events)
     return atomic_fetch_and(&ao->events_, ~(unsigned)events) & events;
 }
 
-static void ao_add_events(struct ao *ao, int events)
+void ao_add_events(struct ao *ao, int events)
 {
     atomic_fetch_or(&ao->events_, events);
     ao->wakeup_cb(ao->wakeup_ctx);
diff --git a/audio/out/ao.h b/audio/out/ao.h
index b9df4ebb00..99a3d0fae0 100644
--- a/audio/out/ao.h
+++ b/audio/out/ao.h
@@ -48,6 +48,7 @@ enum aocontrol {
 enum {
     AO_EVENT_RELOAD = 1,
     AO_EVENT_HOTPLUG = 2,
+    AO_EVENT_INITIAL_UNBLOCK = 4,
 };
 
 enum {
@@ -104,6 +105,8 @@ void ao_resume(struct ao *ao);
 void ao_drain(struct ao *ao);
 bool ao_eof_reached(struct ao *ao);
 int ao_query_and_reset_events(struct ao *ao, int events);
+void ao_add_events(struct ao *ao, int events);
+void ao_unblock(struct ao *ao);
 void ao_request_reload(struct ao *ao);
 void ao_hotplug_event(struct ao *ao);
 
diff --git a/audio/out/ao_lavc.c b/audio/out/ao_lavc.c
index bb86224229..974d9d0b63 100644
--- a/audio/out/ao_lavc.c
+++ b/audio/out/ao_lavc.c
@@ -84,6 +84,13 @@ static void select_format(struct ao *ao, const AVCodec *codec)
     }
 }
 
+static void on_ready(void *ptr)
+{
+    struct ao *ao = ptr;
+
+    ao_add_events(ao, AO_EVENT_INITIAL_UNBLOCK);
+}
+
 // open & setup audio device
 static int init(struct ao *ao)
 {
@@ -123,7 +130,7 @@ static int init(struct ao *ao)
     encoder->sample_fmt = af_to_avformat(ao->format);
     encoder->bits_per_raw_sample = ac->sample_size * 8;
 
-    if (!encoder_init_codec_and_muxer(ac->enc))
+    if (!encoder_init_codec_and_muxer(ac->enc, on_ready, ao))
         goto fail;
 
     ac->pcmhack = 0;
@@ -342,6 +349,7 @@ const struct ao_driver audio_out_lavc = {
     .encode = true,
     .description = "audio encoding using libavcodec",
     .name      = "lavc",
+    .initially_blocked = true,
     .priv_size = sizeof(struct priv),
     .init      = init,
     .uninit    = uninit,
diff --git a/audio/out/internal.h b/audio/out/internal.h
index 33e8a8c6a9..bf769d7e1c 100644
--- a/audio/out/internal.h
+++ b/audio/out/internal.h
@@ -130,6 +130,10 @@ struct ao_driver {
     const char *name;
     // Description shown with --ao=help.
     const char *description;
+    // This requires waiting for a AO_EVENT_INITIAL_UNBLOCK event before the
+    // first play() call is done. Encode mode uses this, and push mode
+    // respects it automatically (don't use with pull mode).
+    bool initially_blocked;
     // Init the device using ao->format/ao->channels/ao->samplerate. If the
     // device doesn't accept these parameters, you can attempt to negotiate
     // fallback parameters, and set the ao format fields accordingly.
diff --git a/audio/out/push.c b/audio/out/push.c
index b198afef91..470f521c68 100644
--- a/audio/out/push.c
+++ b/audio/out/push.c
@@ -56,6 +56,7 @@ struct ao_push_state {
     bool still_playing;
     bool need_wakeup;
     bool paused;
+    bool initial_unblocked;
 
     // Whether the current buffer contains the complete audio.
     bool final_chunk;
@@ -357,7 +358,8 @@ static void *playthread(void *arg)
     mpthread_set_name("ao");
     pthread_mutex_lock(&p->lock);
     while (!p->terminate) {
-        bool playing = !p->paused || ao->stream_silence;
+        bool blocked = ao->driver->initially_blocked && !p->initial_unblocked;
+        bool playing = (!p->paused || ao->stream_silence) && !blocked;
         if (playing)
             ao_play_data(ao);
 
@@ -502,6 +504,19 @@ int ao_play_silence(struct ao *ao, int samples)
     return ao->driver->play(ao, (void **)p->silence, samples, 0);
 }
 
+void ao_unblock(struct ao *ao)
+{
+    if (ao->api == &ao_api_push) {
+        struct ao_push_state *p = ao->api_priv;
+        pthread_mutex_lock(&p->lock);
+        p->need_wakeup = true;
+        p->initial_unblocked = true;
+        wakeup_playthread(ao);
+        pthread_cond_signal(&p->wakeup);
+        pthread_mutex_unlock(&p->lock);
+    }
+}
+
 #ifndef __MINGW32__
 
 #include <poll.h>
diff --git a/common/encode.h b/common/encode.h
index 4a09fedaa2..62a8b094e9 100644
--- a/common/encode.h
+++ b/common/encode.h
@@ -56,6 +56,8 @@ bool encode_lavc_showhelp(struct mp_log *log, struct encode_opts *options);
 int encode_lavc_getstatus(struct encode_lavc_context *ctx, char *buf, int bufsize, float relative_position);
 void encode_lavc_expect_stream(struct encode_lavc_context *ctx,
                                enum stream_type type);
+void encode_lavc_stream_eof(struct encode_lavc_context *ctx,
+                            enum stream_type type);
 void encode_lavc_set_metadata(struct encode_lavc_context *ctx,
                               struct mp_tags *metadata);
 void encode_lavc_set_audio_pts(struct encode_lavc_context *ctx, double pts);
diff --git a/common/encode_lavc.c b/common/encode_lavc.c
index 2473609921..d8e875c58f 100644
--- a/common/encode_lavc.c
+++ b/common/encode_lavc.c
@@ -53,10 +53,6 @@ struct encode_priv {
     struct mux_stream **streams;
     int num_streams;
 
-    // Temporary queue while muxer is not initialized.
-    AVPacket **packets;
-    int num_packets;
-
     // Statistics
     double t0;
 
@@ -69,10 +65,13 @@ struct encode_priv {
 
 struct mux_stream {
     int index;                      // index of this into p->streams[]
+    char name[80];
     struct encode_lavc_context *ctx;
     enum AVMediaType codec_type;
     AVRational encoder_timebase;    // packet timestamps from encoder
     AVStream *st;
+    void (*on_ready)(void *ctx);    // when finishing muxer init
+    void *on_ready_ctx;
 };
 
 #define OPT_BASE_STRUCT struct encode_opts
@@ -112,8 +111,6 @@ const struct m_sub_options encode_config = {
     },
 };
 
-static void write_remaining_packets(struct encode_lavc_context *ctx);
-
 struct encode_lavc_context *encode_lavc_init(struct mpv_global *global)
 {
     struct encode_lavc_context *ctx = talloc_ptrtype(NULL, ctx);
@@ -219,8 +216,6 @@ bool encode_lavc_free(struct encode_lavc_context *ctx)
         p->failed = true;
     }
 
-    write_remaining_packets(ctx);
-
     if (!p->failed && p->header_written) {
         if (av_write_trailer(p->muxer) < 0)
             MP_ERR(p, "error writing trailer\n");
@@ -257,52 +252,6 @@ void encode_lavc_set_audio_pts(struct encode_lavc_context *ctx, double pts)
     }
 }
 
-// called locked
-static void write_remaining_packets(struct encode_lavc_context *ctx)
-{
-    struct encode_priv *p = ctx->priv;
-
-    if (!p->header_written && !p->failed)
-        return; // wait until muxer initialization
-
-    for (int n = 0; n < p->num_packets; n++) {
-        AVPacket *pkt = p->packets[n];
-        MP_TARRAY_REMOVE_AT(p->packets, p->num_packets, 0);
-
-        if (p->failed) {
-            av_packet_free(&pkt);
-            continue;
-        }
-
-        struct mux_stream *s = p->streams[pkt->stream_index];
-
-        pkt->stream_index = s->st->index;
-        assert(s->st == p->muxer->streams[pkt->stream_index]);
-
-        av_packet_rescale_ts(pkt, s->encoder_timebase, s->st->time_base);
-
-        switch (s->st->codecpar->codec_type) {
-        case AVMEDIA_TYPE_VIDEO:
-            p->vbytes += pkt->size;
-            p->frames += 1;
-            break;
-        case AVMEDIA_TYPE_AUDIO:
-            p->abytes += pkt->size;
-            p->audioseconds += pkt->duration
-                * (double)s->st->time_base.num
-                / (double)s->st->time_base.den;
-            break;
-        }
-
-        if (av_interleaved_write_frame(p->muxer, pkt) < 0) {
-            MP_ERR(p, "Writing packet failed.\n");
-            p->failed = true;
-        }
-    }
-
-    p->num_packets = 0;
-}
-
 // called locked
 static void maybe_init_muxer(struct encode_lavc_context *ctx)
 {
@@ -315,15 +264,8 @@ static void maybe_init_muxer(struct encode_lavc_context *ctx)
     // AVStream parameters, so we wait for data from _all_ streams before
     // starting.
     for (int n = 0; n < p->num_streams; n++) {
-        if (!p->streams[n]->st) {
-            int max_num = 50;
-            if (p->num_packets > max_num) {
-                MP_FATAL(p, "no data on stream %d, even though other streams "
-                         "produced more than %d packets.\n", n, p->num_packets);
-                goto failed;
-            }
+        if (!p->streams[n]->st)
             return;
-        }
     }
 
     if (!(p->muxer->oformat->flags & AVFMT_NOFILE)) {
@@ -364,7 +306,13 @@ static void maybe_init_muxer(struct encode_lavc_context *ctx)
 
     p->header_written = true;
 
-    write_remaining_packets(ctx);
+    for (int n = 0; n < p->num_streams; n++) {
+        struct mux_stream *s = p->streams[n];
+
+        if (s->on_ready)
+            s->on_ready(s->on_ready_ctx);
+    }
+
     return;
 
 failed:
@@ -411,17 +359,45 @@ void encode_lavc_expect_stream(struct encode_lavc_context *ctx,
         .ctx = ctx,
         .codec_type = mp_to_av_stream_type(type),
     };
+    snprintf(dst->name, sizeof(dst->name), "%s", stream_type_name(type));
     MP_TARRAY_APPEND(p, p->streams, p->num_streams, dst);
 
 done:
     pthread_mutex_unlock(&ctx->lock);
 }
 
+void encode_lavc_stream_eof(struct encode_lavc_context *ctx,
+                            enum stream_type type)
+{
+    if (!ctx)
+        return;
+
+    struct encode_priv *p = ctx->priv;
+
+    pthread_mutex_lock(&ctx->lock);
+
+    enum AVMediaType codec_type = mp_to_av_stream_type(type);
+    struct mux_stream *dst = find_mux_stream(ctx, codec_type);
+
+    // If we've reached EOF, even though the stream was selected, and we didn't
+    // ever initialize it, we have a problem. We could mux some sort of dummy
+    // stream (and could fail if actual data arrives later), or we bail out
+    // early.
+    if (dst && !dst->st) {
+        MP_ERR(p, "No data on stream %s.\n", dst->name);
+        p->failed = true;
+    }
+
+    pthread_mutex_unlock(&ctx->lock);
+}
+
 // Signal that you are ready to encode (you provide the codec params etc. too).
 // This returns a muxing handle which you can use to add encodec packets.
 // Can be called only once per stream. info is copied by callee as needed.
 static struct mux_stream *encode_lavc_add_stream(struct encode_lavc_context *ctx,
-                                                 struct encoder_stream_info *info)
+                                                 struct encoder_stream_info *info,
+                                                 void (*on_ready)(void *ctx),
+                                                 void *on_ready_ctx)
 {
     struct encode_priv *p = ctx->priv;
 
@@ -449,6 +425,9 @@ static struct mux_stream *encode_lavc_add_stream(struct encode_lavc_context *ctx
     if (avcodec_parameters_copy(dst->st->codecpar, info->codecpar) < 0)
         MP_HANDLE_OOM(0);
 
+    dst->on_ready = on_ready;
+    dst->on_ready_ctx = on_ready_ctx;
+
     maybe_init_muxer(ctx);
 
 done:
@@ -470,17 +449,41 @@ static void encode_lavc_add_packet(struct mux_stream *dst, AVPacket *pkt)
     if (p->failed)
         goto done;
 
-    AVPacket *new_packet = av_packet_clone(pkt);
-    if (pkt && !new_packet)
-        MP_HANDLE_OOM(0);
+    if (!p->header_written) {
+        MP_ERR(p, "Encoder trying to write packet before muxer was initialized.\n");
+        p->failed = true;
+        goto done;
+    }
 
-    new_packet->stream_index = dst->index; // not the lavf index (yet)
-    MP_TARRAY_APPEND(p, p->packets, p->num_packets, new_packet);
+    pkt->stream_index = dst->st->index;
+    assert(dst->st == p->muxer->streams[pkt->stream_index]);
 
-    write_remaining_packets(ctx);
+    av_packet_rescale_ts(pkt, dst->encoder_timebase, dst->st->time_base);
+
+    switch (dst->st->codecpar->codec_type) {
+    case AVMEDIA_TYPE_VIDEO:
+        p->vbytes += pkt->size;
+        p->frames += 1;
+        break;
+    case AVMEDIA_TYPE_AUDIO:
+        p->abytes += pkt->size;
+        p->audioseconds += pkt->duration
+            * (double)dst->st->time_base.num
+            / (double)dst->st->time_base.den;
+        break;
+    }
+
+    if (av_interleaved_write_frame(p->muxer, pkt) < 0) {
+        MP_ERR(p, "Writing packet failed.\n");
+        p->failed = true;
+        goto done;
+    }
+
+    pkt = NULL;
 
 done:
     pthread_mutex_unlock(&ctx->lock);
+    av_packet_free(&pkt);
 }
 
 void encode_lavc_discontinuity(struct encode_lavc_context *ctx)
@@ -852,7 +855,8 @@ static void encoder_2pass_prepare(struct encoder_context *p)
     talloc_free(filename);
 }
 
-bool encoder_init_codec_and_muxer(struct encoder_context *p)
+bool encoder_init_codec_and_muxer(struct encoder_context *p,
+                                  void (*on_ready)(void *ctx), void *ctx)
 {
     assert(!avcodec_is_open(p->encoder));
 
@@ -905,7 +909,8 @@ bool encoder_init_codec_and_muxer(struct encoder_context *p)
     if (avcodec_parameters_from_context(p->info.codecpar, p->encoder) < 0)
         goto fail;
 
-    p->mux_stream = encode_lavc_add_stream(p->encode_lavc_ctx, &p->info);
+    p->mux_stream = encode_lavc_add_stream(p->encode_lavc_ctx, &p->info,
+                                           on_ready, ctx);
     if (!p->mux_stream)
         goto fail;
 
diff --git a/common/encode_lavc.h b/common/encode_lavc.h
index a689b1ea47..97c2cf01f1 100644
--- a/common/encode_lavc.h
+++ b/common/encode_lavc.h
@@ -101,7 +101,12 @@ struct encoder_context *encoder_context_alloc(struct encode_lavc_context *ctx,
 
 // After setting your codec parameters on p->encoder, you call this to "open"
 // the encoder. This also initializes p->mux_stream. Returns false on failure.
-bool encoder_init_codec_and_muxer(struct encoder_context *p);
+// on_ready is called as soon as the muxer has been initialized. Then you are
+// allowed to write packets with encoder_encode().
+// Warning: the on_ready callback is called asynchronously, so you need to
+// make sure to properly synchronize everything.
+bool encoder_init_codec_and_muxer(struct encoder_context *p,
+                                  void (*on_ready)(void *ctx), void *ctx);
 
 // Encode the frame and write the packet. frame is ref'ed as need.
 bool encoder_encode(struct encoder_context *p, AVFrame *frame);
diff --git a/player/audio.c b/player/audio.c
index ce9e1cd06b..9b1340ec96 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -790,6 +790,10 @@ void fill_audio_out_buffers(struct MPContext *mpctx)
     if (mpctx->ao && ao_query_and_reset_events(mpctx->ao, AO_EVENT_RELOAD))
         reload_audio_output(mpctx);
 
+    if (mpctx->ao && ao_query_and_reset_events(mpctx->ao,
+                                               AO_EVENT_INITIAL_UNBLOCK))
+        ao_unblock(mpctx->ao);
+
     struct ao_chain *ao_c = mpctx->ao_chain;
     if (!ao_c)
         return;
@@ -811,6 +815,7 @@ void fill_audio_out_buffers(struct MPContext *mpctx)
             mpctx->audio_status = STATUS_EOF;
             MP_VERBOSE(mpctx, "audio EOF without any data\n");
             mp_filter_reset(ao_c->filter->f);
+            encode_lavc_stream_eof(mpctx->encode_lavc_ctx, STREAM_AUDIO);
         }
         return; // try again next iteration
     }
@@ -994,6 +999,7 @@ void fill_audio_out_buffers(struct MPContext *mpctx)
             if (!was_eof) {
                 MP_VERBOSE(mpctx, "audio EOF reached\n");
                 mp_wakeup_core(mpctx);
+                encode_lavc_stream_eof(mpctx->encode_lavc_ctx, STREAM_AUDIO);
             }
         }
     }
diff --git a/player/video.c b/player/video.c
index 693454ecb0..17dff84984 100644
--- a/player/video.c
+++ b/player/video.c
@@ -1027,6 +1027,7 @@ void write_video(struct MPContext *mpctx)
             if (mpctx->time_frame <= 0) {
                 MP_VERBOSE(mpctx, "video EOF reached\n");
                 mpctx->video_status = STATUS_EOF;
+                encode_lavc_stream_eof(mpctx->encode_lavc_ctx, STREAM_VIDEO);
             }
         }
 
diff --git a/video/out/vo.c b/video/out/vo.c
index 624136bd47..acae4f2acb 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -750,7 +750,9 @@ bool vo_is_ready_for_frame(struct vo *vo, int64_t next_pts)
 {
     struct vo_internal *in = vo->in;
     pthread_mutex_lock(&in->lock);
-    bool r = vo->config_ok && !in->frame_queued &&
+    bool blocked = vo->driver->initially_blocked &&
+                   !(in->internal_events & VO_EVENT_INITIAL_UNBLOCK);
+    bool r = vo->config_ok && !in->frame_queued && !blocked &&
              (!in->current_frame || in->current_frame->num_vsyncs < 1);
     if (r && next_pts >= 0) {
         // Don't show the frame too early - it would basically freeze the
diff --git a/video/out/vo.h b/video/out/vo.h
index 17cb692356..3c00bb988e 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -45,10 +45,13 @@ enum {
     VO_EVENT_LIVE_RESIZING              = 1 << 5,
     // Window fullscreen state changed via external influence.
     VO_EVENT_FULLSCREEN_STATE           = 1 << 6,
+    // Special thing for encode mode (vo_driver.initially_blocked).
+    // Part of VO_EVENTS_USER to make vo_is_ready_for_frame() work properly.
+    VO_EVENT_INITIAL_UNBLOCK            = 1 << 7,
 
     // Set of events the player core may be interested in.
     VO_EVENTS_USER = VO_EVENT_RESIZE | VO_EVENT_WIN_STATE |
-                     VO_EVENT_FULLSCREEN_STATE,
+                     VO_EVENT_FULLSCREEN_STATE | VO_EVENT_INITIAL_UNBLOCK,
 };
 
 enum mp_voctrl {
@@ -264,6 +267,12 @@ struct vo_driver {
     // Encoding functionality, which can be invoked via --o only.
     bool encode;
 
+    // This requires waiting for a VO_EVENT_INITIAL_UNBLOCK event before the
+    // first frame can be sent. Doing vo_reconfig*() calls is allowed though.
+    // Encode mode uses this, the core uses vo_is_ready_for_frame() to
+    // implicitly check for this.
+    bool initially_blocked;
+
     // VO_CAP_* bits
     int caps;
 
diff --git a/video/out/vo_lavc.c b/video/out/vo_lavc.c
index 2f48e3f750..e817b530e0 100644
--- a/video/out/vo_lavc.c
+++ b/video/out/vo_lavc.c
@@ -60,6 +60,13 @@ static void uninit(struct vo *vo)
         encoder_encode(enc, NULL); // finish encoding
 }
 
+static void on_ready(void *ptr)
+{
+    struct vo *vo = ptr;
+
+    vo_event(vo, VO_EVENT_INITIAL_UNBLOCK);
+}
+
 static int reconfig2(struct vo *vo, struct mp_image *img)
 {
     struct priv *vc = vo->priv;
@@ -127,7 +134,7 @@ static int reconfig2(struct vo *vo, struct mp_image *img)
 
     encoder->time_base = av_inv_q(tb);
 
-    if (!encoder_init_codec_and_muxer(vc->enc))
+    if (!encoder_init_codec_and_muxer(vc->enc, on_ready, vo))
         goto error;
 
     return 0;
@@ -233,6 +240,7 @@ const struct vo_driver video_out_lavc = {
     .encode = true,
     .description = "video encoding using libavcodec",
     .name = "lavc",
+    .initially_blocked = true,
     .untimed = true,
     .priv_size = sizeof(struct priv),
     .preinit = preinit,