stream: add a file cache

For remarks, pretty much see the manpage additions. Could help with
network streams that require too much seeking (maybe), or might be
extended to help with the use case of watching and downloading a file
at the same time.

In general, it might be a useless feature and could be removed again.

DOCS/man/options.rst

@@ -428,6 +428,31 @@ OPTIONS
     on the situation, either of these might be slower than the other method.
     This option allows control over this.
 
+``--cache-file=<path>``
+    Create a cache file on the filesystem with the given name. The file is
+    always overwritten. When the general cache is enabled, this file cache
+    will be used to store whatever is read from the source stream.
+
+    This will always overwrite the cache file, and you can't use an existing
+    cache file to resume playback of a stream. (Technically, mpv wouldn't
+    even know which blocks in the file are valid and which not.)
+
+    The resulting file will not necessarily contain all data of the source
+    stream. For example, if you seek, the parts that were skipped over are
+    never read and consequently are not written to the cache. The skipped over
+    parts are filled with zeros. This means that the cache file doesn't
+    necessarily correspond to a full download of the source stream.
+
+    Both of these issues could be improved if there is any user interest.
+
+    Also see ``--cache-file-size``.
+
+``--cache-file-size=<kBytes>``
+    Maximum size of the file created with ``--cache-file``. For read accesses
+    above this size, the cache is simply not used.
+
+    (Default: 1048576, 1 GB.)
+
 ``--cdda-...``
     These options can be used to tune the CD Audio reading feature of mpv.
 

old-makefile

@@ -206,6 +206,7 @@ SOURCES = audio/audio.c \
           player/timeline/tl_mpv_edl.c \
           player/timeline/tl_cue.c \
           stream/cache.c \
+          stream/cache_file.c \
           stream/cookies.c \
           stream/rar.c \
           stream/stream.c \

options/options.c

@@ -139,6 +139,8 @@ const m_option_t mp_opts[] = {
                       ({"no", 0})),
     OPT_INTRANGE("cache-initial", stream_cache.initial, 0, 0, 0x7fffffff),
     OPT_INTRANGE("cache-seek-min", stream_cache.seek_min, 0, 0, 0x7fffffff),
+    OPT_STRING("cache-file", stream_cache.file, 0),
+    OPT_INTRANGE("cache-file-size", stream_cache.file_max, 0, 0, 0x7fffffff),
     OPT_CHOICE_OR_INT("cache-pause-below", stream_cache_pause, 0, 0, 0x7fffffff,
                       ({"no", 0})),
     OPT_INTRANGE("cache-pause-restart", stream_cache_unpause, 0, 0, 0x7fffffff),
@@ -573,6 +575,7 @@ const struct MPOpts mp_default_opts = {
         .def_size = 25000,
         .initial = 0,
         .seek_min = 500,
+        .file_max = 1024 * 1024,
     },
     .stream_cache_pause = 500,
     .stream_cache_unpause = 1000,

options/options.h

@@ -47,6 +47,8 @@ struct mp_cache_opts {
     int def_size;
     int initial;
     int seek_min;
+    char *file;
+    int file_max;
 };
 
 typedef struct MPOpts {

stream/cache_file.c

@@ -0,0 +1,149 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <stdio.h>
+#include <stdint.h>
+
+#include "common/common.h"
+#include "common/msg.h"
+
+#include "options/options.h"
+
+#include "stream.h"
+
+#define BLOCK_SIZE 1024LL
+#define BLOCK_ALIGN(p) ((p) & ~(BLOCK_SIZE - 1))
+
+struct priv {
+    struct stream *original;
+    FILE *cache_file;
+    uint8_t *block_bits;    // 1 bit for each BLOCK_SIZE, whether block was read
+    int64_t size;           // currently known size
+    int64_t max_size;       // max. size for block_bits and cache_file
+};
+
+static bool test_bit(struct priv *p, int64_t pos)
+{
+    if (pos < 0 || pos >= p->size)
+        return false;
+    size_t block = pos / BLOCK_SIZE;
+    return p->block_bits[block / 8] & (1 << (block % 8));
+}
+
+static void set_bit(struct priv *p, int64_t pos, bool bit)
+{
+    if (pos < 0 || pos >= p->size)
+        return;
+    size_t block = pos / BLOCK_SIZE;
+    unsigned int m = (1 << (block % 8));
+    p->block_bits[block / 8] = (p->block_bits[block / 8] & ~m) | (bit ? m : 0);
+}
+
+static int fill_buffer(stream_t *s, char *buffer, int max_len)
+{
+    struct priv *p = s->priv;
+    if (s->pos < 0)
+        return -1;
+    if (s->pos >= p->max_size) {
+        if (stream_seek(p->original, s->pos) < 1)
+            return -1;
+        return stream_read(p->original, buffer, max_len);
+    }
+    // Size of file changes -> invalidate last block
+    if (s->pos >= p->size - BLOCK_SIZE) {
+        int64_t new_size = -1;
+        stream_control(s, STREAM_CTRL_GET_SIZE, &new_size);
+        if (new_size != p->size)
+            set_bit(p, BLOCK_ALIGN(p->size), 0);
+        p->size = MPMIN(p->max_size, new_size);
+    }
+    int64_t aligned = BLOCK_ALIGN(s->pos);
+    if (!test_bit(p, aligned)) {
+        char tmp[BLOCK_SIZE];
+        stream_seek(p->original, aligned);
+        int r = stream_read(p->original, tmp, BLOCK_SIZE);
+        if (r < BLOCK_SIZE) {
+            if (p->size < 0) {
+                MP_WARN(s, "suspected EOF\n");
+            } else if (aligned + r < p->size) {
+                MP_ERR(s, "unexpected EOF\n");
+                return -1;
+            }
+        }
+        if (fseeko(p->cache_file, aligned, SEEK_SET))
+            return -1;
+        if (fwrite(tmp, r, 1, p->cache_file) != 1)
+            return -1;
+        set_bit(p, aligned, 1);
+    }
+    if (fseeko(p->cache_file, s->pos, SEEK_SET))
+        return -1;
+    // align/limit to blocks
+    max_len = MPMIN(max_len, BLOCK_SIZE - (s->pos % BLOCK_SIZE));
+    // Limit to max. known file size
+    max_len = MPMIN(max_len, p->size - s->pos);
+    return fread(buffer, 1, max_len, p->cache_file);
+}
+
+static int seek(stream_t *s, int64_t newpos)
+{
+    return 1;
+}
+
+static int control(stream_t *s, int cmd, void *arg)
+{
+    struct priv *p = s->priv;
+    return stream_control(p->original, cmd, arg);
+}
+
+static void s_close(stream_t *s)
+{
+    struct priv *p = s->priv;
+    if (p->cache_file)
+        fclose(p->cache_file);
+    talloc_free(p);
+}
+
+// return 1 on success, 0 if disabled, -1 on error
+int stream_file_cache_init(stream_t *cache, stream_t *stream,
+                           struct mp_cache_opts *opts)
+{
+    if (!opts->file || !opts->file[0] || opts->file_max < 1)
+        return 0;
+
+    FILE *file = fopen(opts->file, "wb+");
+    if (!file) {
+        MP_ERR(cache, "can't open cache file '%s'\n", opts->file);
+        return -1;
+    }
+
+    struct priv *p = talloc_zero(NULL, struct priv);
+
+    cache->priv = p;
+    p->original = stream;
+    p->cache_file = file;
+    p->max_size = opts->file_max * 1024LL;
+
+    // file_max can be INT_MAX, so this is at most about 256MB
+    p->block_bits = talloc_zero_size(p, (p->max_size / BLOCK_SIZE + 1) / 8 + 1);
+
+    cache->seek = seek;
+    cache->fill_buffer = fill_buffer;
+    cache->control = control;
+    cache->close = s_close;
+
+    return 1;
+}

stream/stream.c

@@ -766,6 +766,28 @@ stream_t *open_memory_stream(void *data, int len)
     return s;
 }
 
+static stream_t *open_cache(stream_t *orig, const char *name)
+{
+    stream_t *cache = new_stream();
+    cache->uncached_type = orig->type;
+    cache->uncached_stream = orig;
+    cache->seekable = true;
+    cache->mode = STREAM_READ;
+    cache->read_chunk = 4 * STREAM_BUFFER_SIZE;
+
+    cache->url = talloc_strdup(cache, orig->url);
+    cache->mime_type = talloc_strdup(cache, orig->mime_type);
+    cache->demuxer = talloc_strdup(cache, orig->demuxer);
+    cache->lavf_type = talloc_strdup(cache, orig->lavf_type);
+    cache->safe_origin = orig->safe_origin;
+    cache->opts = orig->opts;
+    cache->global = orig->global;
+
+    cache->log = mp_log_new(cache, cache->global->log, name);
+
+    return cache;
+}
+
 static struct mp_cache_opts check_cache_opts(stream_t *stream,
                                              struct mp_cache_opts *opts)
 {
@@ -794,27 +816,21 @@ int stream_enable_cache(stream_t **stream, struct mp_cache_opts *opts)
     if (use_opts.size < 1)
         return -1;
 
-    stream_t *cache = new_stream();
-    cache->uncached_type = orig->type;
-    cache->uncached_stream = orig;
-    cache->seekable = true;
-    cache->mode = STREAM_READ;
-    cache->read_chunk = 4 * STREAM_BUFFER_SIZE;
+    stream_t *fcache = open_cache(orig, "file-cache");
+    if (stream_file_cache_init(fcache, orig, &use_opts) <= 0) {
+        fcache->uncached_stream = NULL; // don't free original stream
+        free_stream(fcache);
+        fcache = orig;
+    }
 
-    cache->url = talloc_strdup(cache, orig->url);
-    cache->mime_type = talloc_strdup(cache, orig->mime_type);
-    cache->demuxer = talloc_strdup(cache, orig->demuxer);
-    cache->lavf_type = talloc_strdup(cache, orig->lavf_type);
-    cache->safe_origin = orig->safe_origin;
-    cache->opts = orig->opts;
-    cache->global = orig->global;
+    stream_t *cache = open_cache(fcache, "cache");
 
-    cache->log = mp_log_new(cache, cache->global->log, "cache");
-
-    int res = stream_cache_init(cache, orig, &use_opts);
+    int res = stream_cache_init(cache, fcache, &use_opts);
     if (res <= 0) {
         cache->uncached_stream = NULL; // don't free original stream
         free_stream(cache);
+        if (fcache != orig)
+            free_stream(fcache);
     } else {
         *stream = cache;
     }

stream/stream.h

@@ -198,6 +198,8 @@ int stream_enable_cache(stream_t **stream, struct mp_cache_opts *opts);
 // Internal
 int stream_cache_init(stream_t *cache, stream_t *stream,
                       struct mp_cache_opts *opts);
+int stream_file_cache_init(stream_t *cache, stream_t *stream,
+                           struct mp_cache_opts *opts);
 
 int stream_write_buffer(stream_t *s, unsigned char *buf, int len);
 

wscript_build.py

@@ -233,6 +233,7 @@ def build(ctx):
         ( "stream/ai_sndio.c",                   "sndio" ),
         ( "stream/audio_in.c",                   "audio-input" ),
         ( "stream/cache.c" ),
+        ( "stream/cache_file.c" ),
         ( "stream/cookies.c" ),
         ( "stream/dvb_tune.c",                   "dvbin" ),
         ( "stream/frequencies.c",                "tv" ),