avformat/cache: support non continuous caching

This allows using the cache protocol on top of seekable but slow protocols to
speed them up

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/cache.c

@@ -1,6 +1,6 @@
 /*
  * Input cache protocol.
- * Copyright (c) 2011 Michael Niedermayer
+ * Copyright (c) 2011,2014 Michael Niedermayer
  *
  * This file is part of FFmpeg.
  *
@@ -23,7 +23,6 @@
 
 /**
  * @TODO
- *      support non continuous caching
  *      support keeping files
  *      support filling with a background thread
  */
@@ -31,6 +30,7 @@
 #include "libavutil/avassert.h"
 #include "libavutil/avstring.h"
 #include "libavutil/file.h"
+#include "libavutil/tree.h"
 #include "avformat.h"
 #include <fcntl.h>
 #if HAVE_IO_H
@@ -44,13 +44,28 @@
 #include "os_support.h"
 #include "url.h"
 
+typedef struct CacheEntry {
+    int64_t logical_pos;
+    int64_t physical_pos;
+    int size;
+} CacheEntry;
+
 typedef struct Context {
     int fd;
+    struct AVTreeNode *root;
+    int64_t logical_pos;
+    int64_t cache_pos;
+    int64_t inner_pos;
     int64_t end;
-    int64_t pos;
     URLContext *inner;
+    int64_t cache_hit, cache_miss;
 } Context;
 
+static int cmp(void *key, const void *node)
+{
+    return (*(int64_t *) key) - ((const CacheEntry *) node)->logical_pos;
+}
+
 static int cache_open(URLContext *h, const char *arg, int flags)
 {
     char *buffername;
@@ -70,26 +85,106 @@ static int cache_open(URLContext *h, const char *arg, int flags)
     return ffurl_open(&c->inner, arg, flags, &h->interrupt_callback, NULL);
 }
 
+static int add_entry(URLContext *h, const unsigned char *buf, int size)
+{
+    Context *c= h->priv_data;
+    int64_t pos;
+    int ret;
+    CacheEntry *entry = av_malloc(sizeof(*entry));
+    CacheEntry *entry_ret;
+    struct AVTreeNode *node = av_tree_node_alloc();
+
+    if (!entry || !node) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    //FIXME avoid lseek
+    pos = lseek(c->fd, 0, SEEK_END);
+    if (pos < 0) {
+        ret = AVERROR(errno);
+        av_log(h, AV_LOG_ERROR, "seek in cache failed\n");
+        goto fail;
+    }
+
+    ret = write(c->fd, buf, size);
+    if (ret < 0) {
+        ret = AVERROR(errno);
+        av_log(h, AV_LOG_ERROR, "write in cache failed\n");
+        goto fail;
+    }
+
+    entry->logical_pos = c->logical_pos;
+    entry->physical_pos = pos;
+    entry->size = ret;
+
+    entry_ret = av_tree_insert(&c->root, entry, cmp, &node);
+    if (entry_ret && entry_ret != entry) {
+        ret = -1;
+        av_log(h, AV_LOG_ERROR, "av_tree_insert failed\n");
+        goto fail;
+    }
+    c->cache_pos = entry->physical_pos + entry->size;
+
+    return 0;
+fail:
+    av_free(entry);
+    av_free(node);
+    return ret;
+}
+
 static int cache_read(URLContext *h, unsigned char *buf, int size)
 {
     Context *c= h->priv_data;
+    CacheEntry *entry, *next[2] = {NULL, NULL};
     int r;
 
-    if(c->pos<c->end){
+    entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next);
-        r = read(c->fd, buf, FFMIN(size, c->end - c->pos));
+
-        if(r>0)
+    if (!entry)
-            c->pos += r;
+        entry = next[0];
-        return (-1 == r)?AVERROR(errno):r;
+
-    }else{
+    if (entry) {
-        r = ffurl_read(c->inner, buf, size);
+        int64_t in_block_pos = c->logical_pos - entry->logical_pos;
-        if(r > 0){
+        av_assert0(entry->logical_pos <= c->logical_pos);
-            int r2= write(c->fd, buf, r);
+        if (in_block_pos < entry->size) {
-            av_assert0(r2==r); // FIXME handle cache failure
+            int64_t physical_target = entry->physical_pos + in_block_pos;
-            c->pos += r;
+            //FIXME avoid seek if unneeded
-            c->end += r;
+            r = lseek(c->fd, physical_target, SEEK_SET);
+            if (r >= 0)
+                r = read(c->fd, buf, FFMIN(size, entry->size - in_block_pos));
+
+            if (r > 0) {
+                c->logical_pos += r;
+                c->cache_hit ++;
+                return r;
+            }
         }
-        return r;
     }
+
+    // Cache miss or some kind of fault with the cache
+
+    if (c->logical_pos != c->inner_pos) {
+        r = ffurl_seek(c->inner, c->logical_pos, SEEK_SET);
+        if (r<0) {
+            av_log(h, AV_LOG_ERROR, "Failed to perform internal seek\n");
+            return r;
+        }
+        c->inner_pos = r;
+    }
+
+    r = ffurl_read(c->inner, buf, size);
+    if (r<=0)
+        return r;
+    c->inner_pos += r;
+
+    c->cache_miss ++;
+
+    add_entry(h, buf, r);
+    c->logical_pos += r;
+    c->end = FFMAX(c->end, c->logical_pos);
+
+    return r;
 }
 
 static int64_t cache_seek(URLContext *h, int64_t pos, int whence)
@@ -100,32 +195,45 @@ static int64_t cache_seek(URLContext *h, int64_t pos, int whence)
         pos= ffurl_seek(c->inner, pos, whence);
         if(pos <= 0){
             pos= ffurl_seek(c->inner, -1, SEEK_END);
-            ffurl_seek(c->inner, c->end, SEEK_SET);
+            if (ffurl_seek(c->inner, c->inner_pos, SEEK_SET) < 0)
-            if(pos <= 0)
+                av_log(h, AV_LOG_ERROR, "Inner protocol failed to seekback\n");
-                return c->end;
         }
+        c->end = FFMAX(c->end, pos);
         return pos;
     }
 
-    pos= lseek(c->fd, pos, whence);
+    if (whence == SEEK_CUR) {
-    if(pos<0){
+        whence = SEEK_SET;
-        return pos;
+        pos += c->logical_pos;
-    }else if(pos <= c->end){
-        c->pos= pos;
-        return pos;
-    }else{
-        if(lseek(c->fd, c->pos, SEEK_SET) < 0) {
-            av_log(h, AV_LOG_ERROR, "Failure to seek in cache\n");
-        }
-        return AVERROR(EPIPE);
     }
+
+    if (whence == SEEK_SET && pos >= 0 && pos < c->end) {
+        //Seems within filesize, assume it will not fail.
+        c->logical_pos = pos;
+        return pos;
+    }
+
+    //cache miss
+    pos = lseek(c->fd, pos, whence);
+    if (pos >= 0) {
+        c->logical_pos = pos;
+        c->end = FFMAX(c->end, pos);
+    }
+
+    return pos;
 }
 
 static int cache_close(URLContext *h)
 {
     Context *c= h->priv_data;
+
+    av_log(h, AV_LOG_INFO, "Statistics, cache hits:%"PRId64" cache misses:%"PRId64"\n",
+           c->cache_hit, c->cache_miss);
+
     close(c->fd);
     ffurl_close(c->inner);
+    av_tree_destroy(c->root);
+
 
     return 0;
 }

libavformat/version.h

@@ -31,7 +31,7 @@
 
 #define LIBAVFORMAT_VERSION_MAJOR 56
 #define LIBAVFORMAT_VERSION_MINOR  16
-#define LIBAVFORMAT_VERSION_MICRO 101
+#define LIBAVFORMAT_VERSION_MICRO 102
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
                                                LIBAVFORMAT_VERSION_MINOR, \