demux_libass: change how external ASS subtitles are loaded

Instead of parsing the ASS file in demux_libass.c and trying to pass the
ASS_Track to the subtitle renderer, just read all file data in
demux_libass.c, and let the subtitle renderer pass the file contents to
ass_process_codec_private(). (This happens to parse full files too.)

Makes the code simpler, though it also relies harder on the (messy)
probe logic in demux_libass.c.

demux/demux.h

@@ -156,7 +156,6 @@ struct demuxer_params {
     struct matroska_segment_uid *matroska_wanted_uids;
     int matroska_wanted_segment;
     bool *matroska_was_valid;
-    struct ass_library *ass_library;
     bool expect_subtitle;
 };
 

demux/demux_libass.c

@@ -29,22 +29,26 @@
 
 #define PROBE_SIZE (8 * 1024)
 
-struct priv {
-    ASS_Track *track;
-};
+static void message_callback(int level, const char *format, va_list va, void *ctx)
+{
+    // ignore
+}
 
 static int d_check_file(struct demuxer *demuxer, enum demux_check check)
 {
     const char *user_cp = demuxer->opts->sub_cp;
     struct stream *s = demuxer->stream;
     struct mp_log *log = demuxer->log;
-    // Older versions of libass will behave strange if renderer and track
-    // library handles mismatch, so make sure everything uses a global handle.
-    ASS_Library *lib = demuxer->params ? demuxer->params->ass_library : NULL;
-    if (!lib)
+
+    if (!demuxer->params || !demuxer->params->expect_subtitle)
         return -1;
 
     if (check >= DEMUX_CHECK_UNSAFE) {
+        ASS_Library *lib = ass_library_init();
+        if (!lib)
+            return -1;
+        ass_set_message_cb(lib, message_callback, NULL);
+
         // Probe by loading a part of the beginning of the file with libass.
         // Incomplete scripts are usually ok, and we hope libass is not verbose
         // when dealing with (from its perspective) completely broken binary
@@ -62,12 +66,15 @@ static int d_check_file(struct demuxer *demuxer, enum demux_check check)
         if (cbuf.start == NULL)
             cbuf = buf;
         ASS_Track *track = ass_read_memory(lib, cbuf.start, cbuf.len, NULL);
+        bool ok = !!track;
         if (cbuf.start != buf.start)
             talloc_free(cbuf.start);
         talloc_free(buf.start);
-        if (!track)
+        if (track)
+            ass_free_track(track);
+        ass_library_done(lib);
+        if (!ok)
             return -1;
-        ass_free_track(track);
     }
 
     // Actually load the full thing.
@@ -82,42 +89,22 @@ static int d_check_file(struct demuxer *demuxer, enum demux_check check)
                                                   MP_ICONV_VERBOSE);
     if (cbuf.start == NULL)
         cbuf = buf;
-    ASS_Track *track = ass_read_memory(lib, cbuf.start, cbuf.len, NULL);
     if (cbuf.start != buf.start)
-        talloc_free(cbuf.start);
-    talloc_free(buf.start);
-    if (!track)
-        return -1;
-
-    track->name = strdup(demuxer->filename);
-
-    struct priv *p = talloc_ptrtype(demuxer, p);
-    *p = (struct priv) {
-        .track = track,
-    };
-    demuxer->priv = p;
+        talloc_free(buf.start);
+    talloc_steal(demuxer, cbuf.start);
 
     struct sh_stream *sh = new_sh_stream(demuxer, STREAM_SUB);
-    sh->sub->track = track;
     sh->codec = "ass";
+    sh->sub->extradata = cbuf.start;
+    sh->sub->extradata_len = cbuf.len;
 
     demuxer->seekable = true;
 
     return 0;
 }
 
-static void d_close(struct demuxer *demuxer)
-{
-    struct priv *p = demuxer->priv;
-    if (p) {
-        if (p->track)
-            ass_free_track(p->track);
-    }
-}
-
 const struct demuxer_desc demuxer_desc_libass = {
     .name = "libass",
     .desc = "ASS/SSA subtitles (libass)",
     .open = d_check_file,
-    .close = d_close,
 };

demux/stheader.h

@@ -91,7 +91,6 @@ typedef struct sh_sub {
     double frame_based;         // timestamps are frame-based (and this is the
                                 // fallback framerate used for timestamps)
     bool is_utf8;               // if false, subtitle packet charset is unknown
-    struct ass_track *track;    // loaded by libass
     struct dec_sub *dec_sub;    // decoder context
 } sh_sub_t;
 

player/loadfile.c

@@ -439,9 +439,6 @@ static struct track *add_stream_track(struct MPContext *mpctx,
     };
     MP_TARRAY_APPEND(mpctx, mpctx->tracks, mpctx->num_tracks, track);
 
-    if (stream->type == STREAM_SUB)
-        track->preloaded = !!stream->sub->track;
-
     // Needed for DVD and Blu-ray.
     struct stream *st = track->demuxer->stream;
     if (!track->lang && (st->uncached_type == STREAMTYPE_BLURAY ||
@@ -789,7 +786,6 @@ static struct track *open_external_file(struct MPContext *mpctx, char *filename,
                                 opts->stream_cache_min_percent,
                                 opts->stream_cache_seek_min_percent);
     struct demuxer_params params = {
-        .ass_library = mpctx->ass_library, // demux_libass requires it
         .expect_subtitle = filter == STREAM_SUB,
     };
     struct demuxer *demuxer =

sub/dec_sub.c

@@ -209,7 +209,6 @@ void sub_init_from_sh(struct dec_sub *sub, struct sh_stream *sh)
         sub_set_extradata(sub, sh->sub->extradata, sh->sub->extradata_len);
     struct sd init_sd = sub->init_sd;
     init_sd.codec = sh->codec;
-    init_sd.ass_track = sh->sub->track;
     init_sd.sub_stream_w = sh->sub->w;
     init_sd.sub_stream_h = sh->sub->h;
 
@@ -416,8 +415,7 @@ bool sub_read_all_packets(struct dec_sub *sub, struct sh_stream *sh)
 
     pthread_mutex_lock(&sub->lock);
 
-    if (!sub_accept_packets_in_advance(sub) || sh->sub->track || sub->num_sd < 1)
-    {
+    if (!sub_accept_packets_in_advance(sub) || sub->num_sd < 1) {
         pthread_mutex_unlock(&sub->lock);
         return false;
     }

sub/sd.h

@@ -28,9 +28,6 @@ struct sd {
     // Resolution hints stored in mp4 files.
     int sub_stream_w, sub_stream_h;
 
-    // Make sd_ass use an existing track
-    struct ass_track *ass_track;
-
     // Shared renderer for ASS - done to avoid reloading embedded fonts.
     struct ass_library *ass_library;
     struct ass_renderer *ass_renderer;

sub/sd_ass.c

@@ -68,13 +68,9 @@ static int init(struct sd *sd)
 
     ctx->is_converted = sd->converted_from != NULL;
 
-    if (sd->ass_track) {
-        ctx->ass_track = sd->ass_track;
-    } else {
-        ctx->ass_track = ass_new_track(sd->ass_library);
-        if (!ctx->is_converted)
-            ctx->ass_track->track_type = TRACK_TYPE_ASS;
-    }
+    ctx->ass_track = ass_new_track(sd->ass_library);
+    if (!ctx->is_converted)
+        ctx->ass_track->track_type = TRACK_TYPE_ASS;
 
     if (sd->extradata) {
         ass_process_codec_private(ctx->ass_track, sd->extradata,
@@ -310,8 +306,7 @@ static void uninit(struct sd *sd)
 {
     struct sd_ass_priv *ctx = sd->priv;
 
-    if (sd->ass_track != ctx->ass_track)
-        ass_free_track(ctx->ass_track);
+    ass_free_track(ctx->ass_track);
     talloc_free(ctx);
 }