sub: fix crash with certain uses of --vf=sub

If, for some reason, the subtitle renderer attempts to render a
subtitle before SD_CTRL_SET_VIDEO_PARAMS was called, it passed a
value calculated from invalid values. This can happen with --vf=sub
and --start. The crash happens if 1. there was a subtitle packet that
falls into the timestamp of the rendered video frame, 2. the playloop
hasn't informed the subtitle decoder about the video resolution yet
(normally unneeded, because that is used for weird corner cases only,
so this code is a bit fuzzy), and 3. something actually requests a
frame to be drawn from the subtitle renderer, like with vf_sub.

The actual crash was due to passing NaN as pixel aspect to libass,
which then created glyphs with ridiculous sizes, involving a few
integer overflows and unchecked mallocs.

The sd_lavc.c and sd_spu.c cases probably don't crash, but I'm not
sure, and it's better fix them anyway.

Not bothering with sd_spu.c, this crap is for compatibility and will
be removed soon.

Note that this would have been no problem, had the code checked whether
SD_CTRL_SET_VIDEO_PARAMS was actually called. This commit adds such a
check (although it basically checks after using the parameters).

Regression since 49caa0a7 and 633fde4a.

sub/sd_ass.c

@@ -19,6 +19,7 @@
 #include <stdlib.h>
 #include <assert.h>
 #include <string.h>
+#include <math.h>
 
 #include <libavutil/common.h>
 #include <ass/ass.h>
@@ -142,9 +143,11 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
                                opts->ass_vsfilter_aspect_compat))
     {
         // Let's use the original video PAR for vsfilter compatibility:
-        scale = scale
+        double par = scale
             * (ctx->video_params.d_w / (double)ctx->video_params.d_h)
             / (ctx->video_params.w   / (double)ctx->video_params.h);
+        if (isnormal(par))
+            scale = par;
     }
     mp_ass_configure(renderer, opts, &dim);
     ass_set_aspect_ratio(renderer, scale, 1);

sub/sd_lavc.c

@@ -17,6 +17,7 @@
 
 #include <stdlib.h>
 #include <assert.h>
+#include <math.h>
 
 #include <libavcodec/avcodec.h>
 #include <libavutil/common.h>
@@ -233,9 +234,11 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res d, double pts,
     if (priv->avctx->codec_id == AV_CODEC_ID_DVD_SUBTITLE &&
             opts->stretch_dvd_subs) {
         // For DVD subs, try to keep the subtitle PAR at display PAR.
-        video_par =
+        double par =
               (priv->video_params.d_w / (double)priv->video_params.d_h)
             / (priv->video_params.w   / (double)priv->video_params.h);
+        if (isnormal(par))
+            video_par = par;
     }
     int insize[2];
     get_resolution(sd, insize);

sub/sd_spu.c

@@ -26,7 +26,6 @@
 
 struct sd_spu_priv {
     void *spudec;
-    struct mp_image_params video_params;
 };
 
 static bool is_dvd_sub(const char *t)
@@ -72,22 +71,8 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res d, double pts,
     spudec_set_forced_subs_only(priv->spudec, opts->forced_subs_only);
     spudec_heartbeat(priv->spudec, pts * 90000);
 
-    if (spudec_visible(priv->spudec)) {
-        double xscale = 1;
-        double yscale = 1;
-        if (opts->stretch_dvd_subs) {
-            // For DVD subs, try to keep the subtitle PAR at display PAR.
-            double video_par =
-                  (priv->video_params.d_w / (double)priv->video_params.d_h)
-                / (priv->video_params.w   / (double)priv->video_params.h);
-            if (video_par > 1.0) {
-                xscale /= video_par;
-            } else {
-                yscale *= video_par;
-            }
-        }
-        spudec_get_indexed(priv->spudec, &d, xscale, yscale, res);
-    }
+    if (spudec_visible(priv->spudec))
+        spudec_get_indexed(priv->spudec, &d, 1, 1, res);
 }
 
 static void reset(struct sd *sd)
@@ -105,25 +90,12 @@ static void uninit(struct sd *sd)
     talloc_free(priv);
 }
 
-static int control(struct sd *sd, enum sd_ctrl cmd, void *arg)
-{
-    struct sd_spu_priv *priv = sd->priv;
-    switch (cmd) {
-    case SD_CTRL_SET_VIDEO_PARAMS:
-        priv->video_params = *(struct mp_image_params *)arg;
-        return CONTROL_OK;
-    default:
-        return CONTROL_UNKNOWN;
-    }
-}
-
 const struct sd_functions sd_spu = {
     .name = "spu",
     .supports_format = supports_format,
     .init = init,
     .decode = decode,
     .get_bitmaps = get_bitmaps,
-    .control = control,
     .reset = reset,
     .uninit = uninit,
 };