stream: libarchive wrapper for reading compressed archives

This works similar to the existing .rar support, but uses libarchive.
libarchive supports a number of formats, including zip and (most of)
rar.

Unfortunately, seeking does not work too well. Most libarchive readers
do not support seeking, so it's emulated by skipping data until the
target position. On backwards seek, the file is reopened. This works
fine on a local machine (and if the file is not too large), but will
perform not so well over network connection.

This is disabled by default for now. One reason is that we try
libarchive on every file we open, before trying libavformat, and I'm not
sure if I trust libarchive that much yet. Another reason is that this
breaks multivolume rar support. While libarchive supports seeking in
rar, and (probably) supports multivolume archive, our support of
libarchive (probably) does not. I don't care about multivolume rar, but
vocal users do.

TOOLS/old-configure

@@ -216,6 +216,7 @@ options_state_machine() {
     opt_yes_no _lua         "Lua scripting"
     opt_yes_no _vapoursynth "VapourSynth filter bridge (Python)"
     opt_yes_no _vapoursynth_lazy "VapourSynth filter bridge (Lua)"
+    opt_yes_no _libarchive  "libarchive"
     opt_yes_no _encoding    "encoding functionality" yes
     opt_yes_no _build_man   "building manpage"
 }
@@ -859,6 +860,8 @@ if test "$_vapoursynth" = no && test "$_vapoursynth_lazy" = no ; then
 fi
 check_trivial "VapourSynth core" $_vapoursynth_core VAPOURSYNTH_CORE
 
+check_pkg_config "libarchive support" $_libarchive LIBARCHIVE 'libarchive >= 3.0.0'
+
 check_trivial "encoding" $_encoding ENCODING
 
 # needs dlopen on unix

TOOLS/old-makefile

@@ -108,6 +108,8 @@ SOURCES-$(LIBAVFILTER)          += video/filter/vf_lavfi.c \
 
 SOURCES-$(LUA)                  += player/lua.c
 SOURCES-$(VAPOURSYNTH_CORE)     += video/filter/vf_vapoursynth.c
+SOURCES-$(LIBARCHIVE)           += demux/demux_libarchive.c \
+                                   stream/stream_libarchive.c
 SOURCES-$(DLOPEN)               += video/filter/vf_dlopen.c
 
 SOURCES = audio/audio.c \

demux/demux.c

@@ -53,6 +53,7 @@ extern const demuxer_desc_t demuxer_desc_subreader;
 extern const demuxer_desc_t demuxer_desc_playlist;
 extern const demuxer_desc_t demuxer_desc_disc;
 extern const demuxer_desc_t demuxer_desc_rar;
+extern const demuxer_desc_t demuxer_desc_libarchive;
 
 /* Please do not add any new demuxers here. If you want to implement a new
  * demuxer, add it to libavformat, except for wrappers around external
@@ -71,6 +72,9 @@ const demuxer_desc_t *const demuxer_list[] = {
     &demuxer_desc_libass,
 #endif
     &demuxer_desc_matroska,
+#if HAVE_LIBARCHIVE
+    &demuxer_desc_libarchive,
+#endif
     &demuxer_desc_rar,
     &demuxer_desc_lavf,
     &demuxer_desc_mf,

demux/demux_libarchive.c

@@ -0,0 +1,88 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <archive.h>
+#include <archive_entry.h>
+
+#include "common/common.h"
+#include "common/playlist.h"
+#include "stream/stream.h"
+#include "demux.h"
+
+#include "stream/stream_libarchive.h"
+
+static int cmp_filename(const void *a, const void *b)
+{
+    return strcmp(*(char **)a, *(char **)b);
+}
+
+static int open_file(struct demuxer *demuxer, enum demux_check check)
+{
+    struct mp_archive *mpa = mp_archive_new(demuxer->log, demuxer->stream);
+    if (!mpa)
+        return -1;
+
+    struct playlist *pl = talloc_zero(demuxer, struct playlist);
+    demuxer->playlist = pl;
+
+    // make it load archive://
+    pl->disable_safety = true;
+
+    char *prefix = mp_url_escape(mpa, demuxer->stream->url, "~|");
+
+    char **files = NULL;
+    int num_files = 0;
+
+    for (;;) {
+        struct archive_entry *entry;
+        int r = archive_read_next_header(mpa->arch, &entry);
+        if (r == ARCHIVE_EOF)
+            break;
+        if (r < ARCHIVE_OK)
+            MP_ERR(demuxer, "libarchive: %s\n", archive_error_string(mpa->arch));
+        if (r < ARCHIVE_WARN)
+            break;
+        if (archive_entry_filetype(entry) != AE_IFREG)
+            continue;
+        const char *fn = archive_entry_pathname(entry);
+        // Some archives may have no filenames.
+        if (!fn)
+            fn = talloc_asprintf(mpa, "mpv_unknown#%d\n", num_files);
+        // stream_libarchive.c does the real work
+        char *f = talloc_asprintf(mpa, "archive://%s|%s", prefix, fn);
+        MP_TARRAY_APPEND(mpa, files, num_files, f);
+    }
+
+    if (files)
+        qsort(files, num_files, sizeof(files[0]), cmp_filename);
+
+    for (int n = 0; n < num_files; n++)
+        playlist_add_file(pl, files[n]);
+
+    demuxer->filetype = "archive";
+    demuxer->fully_read = true;
+
+    mp_archive_free(mpa);
+
+    return 0;
+}
+
+const struct demuxer_desc demuxer_desc_libarchive = {
+    .name = "libarchive",
+    .desc = "libarchive wrapper",
+    .open = open_file,
+};

stream/stream.c

@@ -75,6 +75,7 @@ extern const stream_info_t stream_info_bluray;
 extern const stream_info_t stream_info_bdnav;
 extern const stream_info_t stream_info_rar;
 extern const stream_info_t stream_info_edl;
+extern const stream_info_t stream_info_libarchive;
 
 static const stream_info_t *const stream_list[] = {
 #if HAVE_CDDA
@@ -108,6 +109,9 @@ static const stream_info_t *const stream_list[] = {
     &stream_info_bluray,
     &stream_info_bdnav,
 #endif
+#if HAVE_LIBARCHIVE
+    &stream_info_libarchive,
+#endif
 
     &stream_info_memory,
     &stream_info_null,

stream/stream_libarchive.c

@@ -0,0 +1,260 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <archive.h>
+#include <archive_entry.h>
+
+#include "common/common.h"
+#include "stream.h"
+
+#include "stream_libarchive.h"
+
+static ssize_t read_cb(struct archive *arch, void *priv, const void **buffer)
+{
+    struct mp_archive *mpa = priv;
+    int res = stream_read_partial(mpa->src, mpa->buffer, sizeof(mpa->buffer));
+    *buffer = mpa->buffer;
+    return MPMAX(res, 0);
+}
+
+static ssize_t seek_cb(struct archive *arch, void *priv,
+                       int64_t offset, int whence)
+{
+    struct mp_archive *mpa = priv;
+    switch (whence) {
+    case SEEK_SET:
+        break;
+    case SEEK_CUR:
+        offset += mpa->src->pos;
+        break;
+    case SEEK_END: ;
+        int64_t size = -1;
+        stream_control(mpa->src, STREAM_CTRL_GET_SIZE, &size);
+        if (size < 0)
+            return -1;
+        offset += size;
+        break;
+    default:
+        return -1;
+    }
+    return stream_seek(mpa->src, offset) ? offset : -1;
+}
+
+static int64_t skip_cb(struct archive *arch, void *priv, int64_t request)
+{
+    struct mp_archive *mpa = priv;
+    int64_t old = stream_tell(mpa->src);
+    stream_skip(mpa->src, request);
+    return stream_tell(mpa->src) - old;
+}
+
+void mp_archive_free(struct mp_archive *mpa)
+{
+    if (mpa && mpa->arch) {
+        archive_read_close(mpa->arch);
+        archive_read_free(mpa->arch);
+    }
+    talloc_free(mpa);
+}
+
+struct mp_archive *mp_archive_new(struct mp_log *log, struct stream *src)
+{
+    struct mp_archive *mpa = talloc_zero(NULL, struct mp_archive);
+    mpa->src = src;
+    stream_seek(mpa->src, 0);
+    mpa->arch = archive_read_new();
+    if (!mpa->arch)
+        goto err;
+    archive_read_support_format_all(mpa->arch);
+    archive_read_support_filter_all(mpa->arch);
+    archive_read_set_callback_data(mpa->arch, mpa);
+    archive_read_set_read_callback(mpa->arch, read_cb);
+    archive_read_set_skip_callback(mpa->arch, skip_cb);
+    if (mpa->src->seekable)
+        archive_read_set_seek_callback(mpa->arch, seek_cb);
+    if (archive_read_open1(mpa->arch) < ARCHIVE_OK)
+        goto err;
+    return mpa;
+
+err:
+    mp_archive_free(mpa);
+    return NULL;
+}
+
+struct priv {
+    struct mp_archive *mpa;
+    struct stream *src;
+    int64_t entry_size;
+    char *entry_name;
+};
+
+static int reopen_archive(stream_t *s)
+{
+    struct priv *p = s->priv;
+    mp_archive_free(p->mpa);
+    p->mpa = mp_archive_new(s->log, p->src);
+    if (!p->mpa)
+        return STREAM_ERROR;
+
+    // Follows the same logic as demux_libarchive.c.
+    struct mp_archive *mpa = p->mpa;
+    int num_files = 0;
+    for (;;) {
+        struct archive_entry *entry;
+        int r = archive_read_next_header(mpa->arch, &entry);
+        if (r == ARCHIVE_EOF) {
+            MP_ERR(s, "archive entry not found. '%s'\n", p->entry_name);
+            goto error;
+        }
+        if (r < ARCHIVE_OK)
+            MP_ERR(s, "libarchive: %s\n", archive_error_string(mpa->arch));
+        if (r < ARCHIVE_WARN)
+            goto error;
+        if (archive_entry_filetype(entry) != AE_IFREG)
+            continue;
+        const char *fn = archive_entry_pathname(entry);
+        char buf[64];
+        if (!fn) {
+            snprintf(buf, sizeof(buf), "mpv_unknown#%d\n", num_files);
+            fn = buf;
+        }
+        if (strcmp(p->entry_name, fn) == 0) {
+            p->entry_size = -1;
+            if (archive_entry_size_is_set(entry))
+                p->entry_size = archive_entry_size(entry);
+            return STREAM_OK;
+        }
+        num_files++;
+    }
+
+error:
+    mp_archive_free(p->mpa);
+    p->mpa = NULL;
+    MP_ERR(s, "could not open archive\n");
+    return STREAM_ERROR;
+}
+
+static int archive_entry_fill_buffer(stream_t *s, char *buffer, int max_len)
+{
+    struct priv *p = s->priv;
+    if (!p->mpa)
+        return 0;
+    int r = archive_read_data(p->mpa->arch, buffer, max_len);
+    if (r < 0)
+        MP_ERR(s, "libarchive: %s\n", archive_error_string(p->mpa->arch));
+    return r;
+}
+
+static int archive_entry_seek(stream_t *s, int64_t newpos)
+{
+    struct priv *p = s->priv;
+    if (!p->mpa)
+        return -1;
+    if (archive_seek_data(p->mpa->arch, newpos, SEEK_SET) >= 0)
+        return 1;
+    // libarchive can't seek in most formats.
+    if (newpos < s->pos) {
+        // Hack seeking backwards into working by reopening the archive and
+        // starting over.
+        MP_VERBOSE(s, "trying to reopen archive for performing seek\n");
+        if (reopen_archive(s) < STREAM_OK)
+            return -1;
+        s->pos = 0;
+    }
+    if (newpos > s->pos) {
+        // For seeking forwards, just keep reading data (there's no libarchive
+        // skip function either).
+        char buffer[4096];
+        while (newpos > s->pos) {
+            int size = MPMIN(newpos - s->pos, sizeof(buffer));
+            int r = archive_read_data(p->mpa->arch, buffer, size);
+            if (r < 0) {
+                MP_ERR(s, "libarchive: %s\n", archive_error_string(p->mpa->arch));
+                return -1;
+            }
+            s->pos += r;
+        }
+    }
+    return 1;
+}
+
+static void archive_entry_close(stream_t *s)
+{
+    struct priv *p = s->priv;
+    mp_archive_free(p->mpa);
+    free_stream(p->src);
+}
+
+static int archive_entry_control(stream_t *s, int cmd, void *arg)
+{
+    struct priv *p = s->priv;
+    switch (cmd) {
+    case STREAM_CTRL_GET_BASE_FILENAME:
+        *(char **)arg = talloc_strdup(NULL, p->src->url);
+        return STREAM_OK;
+    case STREAM_CTRL_GET_SIZE:
+        if (p->entry_size < 0)
+            break;
+        *(int64_t *)arg = p->entry_size;
+        return STREAM_OK;
+    }
+    return STREAM_UNSUPPORTED;
+}
+
+static int archive_entry_open(stream_t *stream)
+{
+    struct priv *p = talloc_zero(stream, struct priv);
+    stream->priv = p;
+
+    if (!strchr(stream->path, '|'))
+        return STREAM_ERROR;
+
+    char *base = talloc_strdup(p, stream->path);
+    char *name = strchr(base, '|');
+    *name++ = '\0';
+    p->entry_name = name;
+    mp_url_unescape_inplace(base);
+
+    p->src = stream_create(base, STREAM_READ | STREAM_SAFE_ONLY,
+                           stream->cancel, stream->global);
+    if (!p->src) {
+        archive_entry_close(stream);
+        return STREAM_ERROR;
+    }
+
+    int r = reopen_archive(stream);
+    if (r < STREAM_OK) {
+        archive_entry_close(stream);
+        return r;
+    }
+
+    stream->fill_buffer = archive_entry_fill_buffer;
+    if (p->src->seekable) {
+        stream->seek = archive_entry_seek;
+        stream->seekable = true;
+    }
+    stream->close = archive_entry_close;
+    stream->control = archive_entry_control;
+
+    return STREAM_OK;
+}
+
+const stream_info_t stream_info_libarchive = {
+    .name = "libarchive",
+    .open = archive_entry_open,
+    .protocols = (const char*const[]){ "archive", NULL },
+};

stream/stream_libarchive.h

@@ -0,0 +1,10 @@
+struct mp_log;
+
+struct mp_archive {
+    struct archive *arch;
+    struct stream *src;
+    char buffer[4096];
+};
+
+void mp_archive_free(struct mp_archive *mpa);
+struct mp_archive *mp_archive_new(struct mp_log *log, struct stream *src);

wscript

@@ -377,6 +377,11 @@ iconv support use --disable-iconv.",
         'desc': 'VapourSynth filter bridge (Lazy Lua)',
         'deps': ['vapoursynth-core', 'lua'],
         'func': check_true,
+    }, {
+        'name': '--libarchive',
+        'desc': 'libarchive wrapper for reading zip files and more',
+        'func': check_pkg_config('libarchive >= 3.0.0'),
+        'default': 'disable',
     }
 ]
 

wscript_build.py

@@ -170,6 +170,7 @@ def build(ctx):
         ( "demux/demux_disc.c" ),
         ( "demux/demux_edl.c" ),
         ( "demux/demux_lavf.c" ),
+        ( "demux/demux_libarchive.c",            "libarchive" ),
         ( "demux/demux_libass.c",                "libass"),
         ( "demux/demux_mf.c" ),
         ( "demux/demux_mkv.c" ),
@@ -247,6 +248,7 @@ def build(ctx):
         ( "stream/stream_edl.c" ),
         ( "stream/stream_file.c" ),
         ( "stream/stream_lavf.c" ),
+        ( "stream/stream_libarchive.c",          "libarchive" ),
         ( "stream/stream_memory.c" ),
         ( "stream/stream_mf.c" ),
         ( "stream/stream_null.c" ),