encode: disable unsupported media types automatically

If you encode to e.g. an audio-only format, then video is disabled
automatically. This also takes care of the very cryptic error message.
It says "[vo/lavc] codec for video not found". Sort of true, but
obscures the real problem if it's e.g. an audio-only format.

common/encode.h

@@ -54,6 +54,8 @@ bool encode_lavc_free(struct encode_lavc_context *ctx);
 void encode_lavc_discontinuity(struct encode_lavc_context *ctx);
 bool encode_lavc_showhelp(struct mp_log *log, struct encode_opts *options);
 int encode_lavc_getstatus(struct encode_lavc_context *ctx, char *buf, int bufsize, float relative_position);
+bool encode_lavc_stream_type_ok(struct encode_lavc_context *ctx,
+                                enum stream_type type);
 void encode_lavc_expect_stream(struct encode_lavc_context *ctx,
                                enum stream_type type);
 void encode_lavc_set_metadata(struct encode_lavc_context *ctx,

common/encode_lavc.c

@@ -735,6 +735,47 @@ static void encoder_destroy(void *ptr)
     free_stream(p->twopass_bytebuffer);
 }
 
+static AVCodec *find_codec_for(struct encode_lavc_context *ctx,
+                               enum stream_type type, bool *used_auto)
+{
+    char *codec_name = type == STREAM_VIDEO
+        ? ctx->options->vcodec
+        : ctx->options->acodec;
+    enum AVMediaType codec_type = mp_to_av_stream_type(type);
+    const char *tname = stream_type_name(type);
+
+    *used_auto = !(codec_name && codec_name[0]);
+
+    AVCodec *codec;
+    if (*used_auto) {
+        codec = avcodec_find_encoder(av_guess_codec(ctx->oformat, NULL,
+                                     ctx->options->file, NULL,
+                                     codec_type));
+    } else {
+        codec = avcodec_find_encoder_by_name(codec_name);
+        if (!codec)
+            MP_FATAL(ctx, "codec '%s' not found.\n", codec_name);
+    }
+
+    if (codec && codec->type != codec_type) {
+        MP_FATAL(ctx, "codec for %s has wrong media type\n", tname);
+        codec = NULL;
+    }
+
+    return codec;
+}
+
+// Return whether the stream type is "supposed" to work.
+bool encode_lavc_stream_type_ok(struct encode_lavc_context *ctx,
+                                enum stream_type type)
+{
+    // If a codec was forced, let it proceed to actual encoding, and then error
+    // if it doesn't work. (Worried that av_guess_codec() may return NULL for
+    // some formats where a specific codec works anyway.)
+    bool auto_codec;
+    return !!find_codec_for(ctx, type, &auto_codec) || !auto_codec;
+}
+
 struct encoder_context *encoder_context_alloc(struct encode_lavc_context *ctx,
                                               enum stream_type type,
                                               struct mp_log *log)
@@ -755,29 +796,15 @@ struct encoder_context *encoder_context_alloc(struct encode_lavc_context *ctx,
         .encode_lavc_ctx = ctx,
     };
 
-    char *codec_name = type == STREAM_VIDEO
+    bool auto_codec;
-        ? p->options->vcodec
+    AVCodec *codec = find_codec_for(ctx, type, &auto_codec);
-        : p->options->acodec;
-    enum AVMediaType codec_type = mp_to_av_stream_type(type);
     const char *tname = stream_type_name(type);
 
-    AVCodec *codec;
-    if (codec_name&& codec_name[0]) {
-        codec = avcodec_find_encoder_by_name(codec_name);
-    } else {
-        codec = avcodec_find_encoder(av_guess_codec(p->oformat, NULL,
-                                                    p->options->file, NULL,
-                                                    codec_type));
-    }
-
     if (!codec) {
+        if (auto_codec)
             MP_FATAL(p, "codec for %s not found\n", tname);
         goto fail;
     }
-    if (codec->type != codec_type) {
-        MP_FATAL(p, "codec for %s has wrong media type\n", tname);
-        goto fail;
-    }
 
     p->encoder = avcodec_alloc_context3(codec);
     MP_HANDLE_OOM(p->encoder);

player/loadfile.c

@@ -1548,7 +1548,16 @@ static void play_current_file(struct MPContext *mpctx)
             // One track can strictly feed at most 1 decoder
             struct track *track = mpctx->current_track[i][t];
             if (track) {
-                if (track->selected) {
+                if (track->type != STREAM_SUB &&
+                    mpctx->encode_lavc_ctx &&
+                    !encode_lavc_stream_type_ok(mpctx->encode_lavc_ctx,
+                                                track->type))
+                {
+                    MP_WARN(mpctx, "Disabling %s (not supported by target "
+                            "format).\n", stream_type_name(track->type));
+                    mpctx->current_track[i][t] = NULL;
+                    mark_track_selection(mpctx, i, t, -2); // disable
+                } else if (track->selected) {
                     MP_ERR(mpctx, "Track %d can't be selected twice.\n",
                            track->user_tid);
                     mpctx->current_track[i][t] = NULL;