From 0c0168a2104b9f7020e82059121679d302cefa40 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 25 Dec 2014 18:48:05 +0100 Subject: [PATCH] avformat/cache: support non continuous caching This allows using the cache protocol on top of seekable but slow protocols to speed them up Signed-off-by: Michael Niedermayer --- libavformat/cache.c | 168 ++++++++++++++++++++++++++++++++++-------- libavformat/version.h | 2 +- 2 files changed, 139 insertions(+), 31 deletions(-) diff --git a/libavformat/cache.c b/libavformat/cache.c index 35f81e83c1..ef33d173cf 100644 --- a/libavformat/cache.c +++ b/libavformat/cache.c @@ -1,6 +1,6 @@ /* * Input cache protocol. - * Copyright (c) 2011 Michael Niedermayer + * Copyright (c) 2011,2014 Michael Niedermayer * * This file is part of FFmpeg. * @@ -23,7 +23,6 @@ /** * @TODO - * support non continuous caching * support keeping files * support filling with a background thread */ @@ -31,6 +30,7 @@ #include "libavutil/avassert.h" #include "libavutil/avstring.h" #include "libavutil/file.h" +#include "libavutil/tree.h" #include "avformat.h" #include #if HAVE_IO_H @@ -44,13 +44,28 @@ #include "os_support.h" #include "url.h" +typedef struct CacheEntry { + int64_t logical_pos; + int64_t physical_pos; + int size; +} CacheEntry; + typedef struct Context { int fd; + struct AVTreeNode *root; + int64_t logical_pos; + int64_t cache_pos; + int64_t inner_pos; int64_t end; - int64_t pos; URLContext *inner; + int64_t cache_hit, cache_miss; } Context; +static int cmp(void *key, const void *node) +{ + return (*(int64_t *) key) - ((const CacheEntry *) node)->logical_pos; +} + static int cache_open(URLContext *h, const char *arg, int flags) { char *buffername; @@ -70,26 +85,106 @@ static int cache_open(URLContext *h, const char *arg, int flags) return ffurl_open(&c->inner, arg, flags, &h->interrupt_callback, NULL); } +static int add_entry(URLContext *h, const unsigned char *buf, int size) +{ + Context *c= h->priv_data; + int64_t pos; + int ret; + CacheEntry *entry = av_malloc(sizeof(*entry)); + CacheEntry *entry_ret; + struct AVTreeNode *node = av_tree_node_alloc(); + + if (!entry || !node) { + ret = AVERROR(ENOMEM); + goto fail; + } + + //FIXME avoid lseek + pos = lseek(c->fd, 0, SEEK_END); + if (pos < 0) { + ret = AVERROR(errno); + av_log(h, AV_LOG_ERROR, "seek in cache failed\n"); + goto fail; + } + + ret = write(c->fd, buf, size); + if (ret < 0) { + ret = AVERROR(errno); + av_log(h, AV_LOG_ERROR, "write in cache failed\n"); + goto fail; + } + + entry->logical_pos = c->logical_pos; + entry->physical_pos = pos; + entry->size = ret; + + entry_ret = av_tree_insert(&c->root, entry, cmp, &node); + if (entry_ret && entry_ret != entry) { + ret = -1; + av_log(h, AV_LOG_ERROR, "av_tree_insert failed\n"); + goto fail; + } + c->cache_pos = entry->physical_pos + entry->size; + + return 0; +fail: + av_free(entry); + av_free(node); + return ret; +} + static int cache_read(URLContext *h, unsigned char *buf, int size) { Context *c= h->priv_data; + CacheEntry *entry, *next[2] = {NULL, NULL}; int r; - if(c->posend){ - r = read(c->fd, buf, FFMIN(size, c->end - c->pos)); - if(r>0) - c->pos += r; - return (-1 == r)?AVERROR(errno):r; - }else{ - r = ffurl_read(c->inner, buf, size); - if(r > 0){ - int r2= write(c->fd, buf, r); - av_assert0(r2==r); // FIXME handle cache failure - c->pos += r; - c->end += r; + entry = av_tree_find(c->root, &c->logical_pos, cmp, (void**)next); + + if (!entry) + entry = next[0]; + + if (entry) { + int64_t in_block_pos = c->logical_pos - entry->logical_pos; + av_assert0(entry->logical_pos <= c->logical_pos); + if (in_block_pos < entry->size) { + int64_t physical_target = entry->physical_pos + in_block_pos; + //FIXME avoid seek if unneeded + r = lseek(c->fd, physical_target, SEEK_SET); + if (r >= 0) + r = read(c->fd, buf, FFMIN(size, entry->size - in_block_pos)); + + if (r > 0) { + c->logical_pos += r; + c->cache_hit ++; + return r; + } } - return r; } + + // Cache miss or some kind of fault with the cache + + if (c->logical_pos != c->inner_pos) { + r = ffurl_seek(c->inner, c->logical_pos, SEEK_SET); + if (r<0) { + av_log(h, AV_LOG_ERROR, "Failed to perform internal seek\n"); + return r; + } + c->inner_pos = r; + } + + r = ffurl_read(c->inner, buf, size); + if (r<=0) + return r; + c->inner_pos += r; + + c->cache_miss ++; + + add_entry(h, buf, r); + c->logical_pos += r; + c->end = FFMAX(c->end, c->logical_pos); + + return r; } static int64_t cache_seek(URLContext *h, int64_t pos, int whence) @@ -100,32 +195,45 @@ static int64_t cache_seek(URLContext *h, int64_t pos, int whence) pos= ffurl_seek(c->inner, pos, whence); if(pos <= 0){ pos= ffurl_seek(c->inner, -1, SEEK_END); - ffurl_seek(c->inner, c->end, SEEK_SET); - if(pos <= 0) - return c->end; + if (ffurl_seek(c->inner, c->inner_pos, SEEK_SET) < 0) + av_log(h, AV_LOG_ERROR, "Inner protocol failed to seekback\n"); } + c->end = FFMAX(c->end, pos); return pos; } - pos= lseek(c->fd, pos, whence); - if(pos<0){ - return pos; - }else if(pos <= c->end){ - c->pos= pos; - return pos; - }else{ - if(lseek(c->fd, c->pos, SEEK_SET) < 0) { - av_log(h, AV_LOG_ERROR, "Failure to seek in cache\n"); - } - return AVERROR(EPIPE); + if (whence == SEEK_CUR) { + whence = SEEK_SET; + pos += c->logical_pos; } + + if (whence == SEEK_SET && pos >= 0 && pos < c->end) { + //Seems within filesize, assume it will not fail. + c->logical_pos = pos; + return pos; + } + + //cache miss + pos = lseek(c->fd, pos, whence); + if (pos >= 0) { + c->logical_pos = pos; + c->end = FFMAX(c->end, pos); + } + + return pos; } static int cache_close(URLContext *h) { Context *c= h->priv_data; + + av_log(h, AV_LOG_INFO, "Statistics, cache hits:%"PRId64" cache misses:%"PRId64"\n", + c->cache_hit, c->cache_miss); + close(c->fd); ffurl_close(c->inner); + av_tree_destroy(c->root); + return 0; } diff --git a/libavformat/version.h b/libavformat/version.h index 95d84fe4c6..a4b9a43f20 100644 --- a/libavformat/version.h +++ b/libavformat/version.h @@ -31,7 +31,7 @@ #define LIBAVFORMAT_VERSION_MAJOR 56 #define LIBAVFORMAT_VERSION_MINOR 16 -#define LIBAVFORMAT_VERSION_MICRO 101 +#define LIBAVFORMAT_VERSION_MICRO 102 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \ LIBAVFORMAT_VERSION_MINOR, \