http: honor response headers in redirect caching

add a dictionary that maps "src_url" -> "expiry;dst_url", the dictionary
is checked before issuing an http request, and updated after getting a
3xx redirect response.

the cache expiry is determined according to the following (in desc
priority) -
1. Expires header
2. Cache-Control containing no-cache/no-store (disables caching)
3. Cache-Control s-maxage/max-age
4. Http codes 301/308 are cached indefinitely, other codes are not
   cached

libavformat/http.c

@@ -48,6 +48,7 @@
  * path names). */
 #define BUFFER_SIZE   (MAX_URL_SIZE + HTTP_HEADERS_SIZE)
 #define MAX_REDIRECTS 8
+#define MAX_CACHED_REDIRECTS 32
 #define HTTP_SINGLE   1
 #define HTTP_MUTLI    2
 #define MAX_EXPIRY    19
@@ -129,6 +130,9 @@ typedef struct HTTPContext {
     HandshakeState handshake_step;
     int is_connected_server;
     int short_seek_size;
+    int64_t expires;
+    char *new_location;
+    AVDictionary *redirect_cache;
 } HTTPContext;
 
 #define OFFSET(x) offsetof(HTTPContext, x)
@@ -177,8 +181,8 @@ static const AVOption options[] = {
 
 static int http_connect(URLContext *h, const char *path, const char *local_path,
                         const char *hoststr, const char *auth,
-                        const char *proxyauth, int *new_location);
+                        const char *proxyauth);
-static int http_read_header(URLContext *h, int *new_location);
+static int http_read_header(URLContext *h);
 static int http_shutdown(URLContext *h, int flags);
 
 void ff_http_init_auth_state(URLContext *dest, const URLContext *src)
@@ -199,7 +203,7 @@ static int http_open_cnx_internal(URLContext *h, AVDictionary **options)
     char auth[1024], proxyauth[1024] = "";
     char path1[MAX_URL_SIZE], sanitized_path[MAX_URL_SIZE + 1];
     char buf[1024], urlbuf[MAX_URL_SIZE];
-    int port, use_proxy, err, location_changed = 0;
+    int port, use_proxy, err;
     HTTPContext *s = h->priv_data;
 
     av_url_split(proto, sizeof(proto), auth, sizeof(auth),
@@ -259,12 +263,8 @@ static int http_open_cnx_internal(URLContext *h, AVDictionary **options)
             return err;
     }
 
-    err = http_connect(h, path, local_path, hoststr,
+    return http_connect(h, path, local_path, hoststr,
-                       auth, proxyauth, &location_changed);
+                        auth, proxyauth);
-    if (err < 0)
-        return err;
-
-    return location_changed;
 }
 
 static int http_should_reconnect(HTTPContext *s, int err)
@@ -300,31 +300,87 @@ static int http_should_reconnect(HTTPContext *s, int err)
     return av_match_list(http_code, s->reconnect_on_http_error, ',') > 0;
 }
 
+static char *redirect_cache_get(HTTPContext *s)
+{
+    AVDictionaryEntry *re;
+    int64_t expiry;
+    char *delim;
+
+    re = av_dict_get(s->redirect_cache, s->location, NULL, AV_DICT_MATCH_CASE);
+    if (!re) {
+        return NULL;
+    }
+
+    delim = strchr(re->value, ';');
+    if (!delim) {
+        return NULL;
+    }
+
+    expiry = strtoll(re->value, NULL, 10);
+    if (time(NULL) > expiry) {
+        return NULL;
+    }
+
+    return delim + 1;
+}
+
+static int redirect_cache_set(HTTPContext *s, const char *source, const char *dest, int64_t expiry)
+{
+    char *value;
+    int ret;
+
+    value = av_asprintf("%"PRIi64";%s", expiry, dest);
+    if (!value) {
+        return AVERROR(ENOMEM);
+    }
+
+    ret = av_dict_set(&s->redirect_cache, source, value, AV_DICT_MATCH_CASE | AV_DICT_DONT_STRDUP_VAL);
+    if (ret < 0) {
+        av_free(value);
+        return ret;
+    }
+
+    return 0;
+}
+
 /* return non zero if error */
 static int http_open_cnx(URLContext *h, AVDictionary **options)
 {
     HTTPAuthType cur_auth_type, cur_proxy_auth_type;
     HTTPContext *s = h->priv_data;
-    int location_changed, attempts = 0, redirects = 0;
+    int ret, attempts = 0, redirects = 0;
     int reconnect_delay = 0;
     uint64_t off;
+    char *cached;
 
 redo:
+
+    cached = redirect_cache_get(s);
+    if (cached) {
+        av_free(s->location);
+        s->location = av_strdup(cached);
+        if (!s->location) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+        goto redo;
+    }
+
     av_dict_copy(options, s->chained_options, 0);
 
     cur_auth_type       = s->auth_state.auth_type;
     cur_proxy_auth_type = s->auth_state.auth_type;
 
     off = s->off;
-    location_changed = http_open_cnx_internal(h, options);
+    ret = http_open_cnx_internal(h, options);
-    if (location_changed < 0) {
+    if (ret < 0) {
-        if (!http_should_reconnect(s, location_changed) ||
+        if (!http_should_reconnect(s, ret) ||
             reconnect_delay > s->reconnect_delay_max)
             goto fail;
 
         av_log(h, AV_LOG_WARNING, "Will reconnect at %"PRIu64" in %d second(s).\n", off, reconnect_delay);
-        location_changed = ff_network_sleep_interruptible(1000U * 1000 * reconnect_delay, &h->interrupt_callback);
+        ret = ff_network_sleep_interruptible(1000U * 1000 * reconnect_delay, &h->interrupt_callback);
-        if (location_changed != AVERROR(ETIMEDOUT))
+        if (ret != AVERROR(ETIMEDOUT))
             goto fail;
         reconnect_delay = 1 + 2 * reconnect_delay;
 
@@ -354,16 +410,28 @@ redo:
     }
     if ((s->http_code == 301 || s->http_code == 302 ||
          s->http_code == 303 || s->http_code == 307 || s->http_code == 308) &&
-        location_changed == 1) {
+        s->new_location) {
         /* url moved, get next */
         ffurl_closep(&s->hd);
         if (redirects++ >= MAX_REDIRECTS)
             return AVERROR(EIO);
+
+        if (!s->expires) {
+            s->expires = (s->http_code == 301 || s->http_code == 308) ? INT64_MAX : -1;
+        }
+
+        if (s->expires > time(NULL) && av_dict_count(s->redirect_cache) < MAX_CACHED_REDIRECTS) {
+            redirect_cache_set(s, s->location, s->new_location, s->expires);
+        }
+
+        av_free(s->location);
+        s->location = s->new_location;
+        s->new_location = NULL;
+
         /* Restart the authentication process with the new target, which
          * might use a different auth mechanism. */
         memset(&s->auth_state, 0, sizeof(s->auth_state));
         attempts         = 0;
-        location_changed = 0;
         goto redo;
     }
     return 0;
@@ -371,8 +439,8 @@ redo:
 fail:
     if (s->hd)
         ffurl_closep(&s->hd);
-    if (location_changed < 0)
+    if (ret < 0)
-        return location_changed;
+        return ret;
     return ff_http_averror(s->http_code, AVERROR(EIO));
 }
 int ff_http_get_shutdown_status(URLContext *h)
@@ -554,7 +622,7 @@ static void handle_http_errors(URLContext *h, int error)
 
 static int http_handshake(URLContext *c)
 {
-    int ret, err, new_location;
+    int ret, err;
     HTTPContext *ch = c->priv_data;
     URLContext *cl = ch->hd;
     switch (ch->handshake_step) {
@@ -569,7 +637,7 @@ static int http_handshake(URLContext *c)
         return 2;
     case READ_HEADERS:
         av_log(c, AV_LOG_TRACE, "Read headers\n");
-        if ((err = http_read_header(c, &new_location)) < 0) {
+        if ((err = http_read_header(c)) < 0) {
             handle_http_errors(c, err);
             return err;
         }
@@ -666,6 +734,8 @@ bail_out:
     if (ret < 0) {
         av_dict_free(&s->chained_options);
         av_dict_free(&s->cookie_dict);
+        av_dict_free(&s->redirect_cache);
+        av_freep(&s->new_location);
         av_freep(&s->uri);
     }
     return ret;
@@ -753,14 +823,13 @@ static int check_http_code(URLContext *h, int http_code, const char *end)
 
 static int parse_location(HTTPContext *s, const char *p)
 {
-    char redirected_location[MAX_URL_SIZE], *new_loc;
+    char redirected_location[MAX_URL_SIZE];
     ff_make_absolute_url(redirected_location, sizeof(redirected_location),
                          s->location, p);
-    new_loc = av_strdup(redirected_location);
+    av_freep(&s->new_location);
-    if (!new_loc)
+    s->new_location = av_strdup(redirected_location);
+    if (!s->new_location)
         return AVERROR(ENOMEM);
-    av_free(s->location);
-    s->location = new_loc;
     return 0;
 }
 
@@ -983,8 +1052,43 @@ static int cookie_string(AVDictionary *dict, char **cookies)
     return 0;
 }
 
-static int process_line(URLContext *h, char *line, int line_count,
+static void parse_expires(HTTPContext *s, const char *p)
-                        int *new_location)
+{
+    struct tm tm;
+
+    if (!parse_set_cookie_expiry_time(p, &tm)) {
+        s->expires = av_timegm(&tm);
+    }
+}
+
+static void parse_cache_control(HTTPContext *s, const char *p)
+{
+    char *age;
+    int offset;
+
+    /* give 'Expires' higher priority over 'Cache-Control' */
+    if (s->expires) {
+        return;
+    }
+
+    if (av_stristr(p, "no-cache") || av_stristr(p, "no-store")) {
+        s->expires = -1;
+        return;
+    }
+
+    age = av_stristr(p, "s-maxage=");
+    offset = 9;
+    if (!age) {
+        age = av_stristr(p, "max-age=");
+        offset = 8;
+    }
+
+    if (age) {
+        s->expires = time(NULL) + atoi(p + offset);
+    }
+}
+
+static int process_line(URLContext *h, char *line, int line_count)
 {
     HTTPContext *s = h->priv_data;
     const char *auto_method =  h->flags & AVIO_FLAG_READ ? "POST" : "GET";
@@ -1081,7 +1185,6 @@ static int process_line(URLContext *h, char *line, int line_count,
         if (!av_strcasecmp(tag, "Location")) {
             if ((ret = parse_location(s, p)) < 0)
                 return ret;
-            *new_location = 1;
         } else if (!av_strcasecmp(tag, "Content-Length") &&
                    s->filesize == UINT64_MAX) {
             s->filesize = strtoull(p, NULL, 10);
@@ -1124,6 +1227,10 @@ static int process_line(URLContext *h, char *line, int line_count,
         } else if (!av_strcasecmp(tag, "Content-Encoding")) {
             if ((ret = parse_content_encoding(h, p)) < 0)
                 return ret;
+        } else if (!av_strcasecmp(tag, "Expires")) {
+            parse_expires(s, p);
+        } else if (!av_strcasecmp(tag, "Cache-Control")) {
+            parse_cache_control(s, p);
         }
     }
     return 1;
@@ -1229,12 +1336,14 @@ static inline int has_header(const char *str, const char *header)
     return av_stristart(str, header + 2, NULL) || av_stristr(str, header);
 }
 
-static int http_read_header(URLContext *h, int *new_location)
+static int http_read_header(URLContext *h)
 {
     HTTPContext *s = h->priv_data;
     char line[MAX_URL_SIZE];
     int err = 0;
 
+    av_freep(&s->new_location);
+    s->expires = 0;
     s->chunksize = UINT64_MAX;
 
     for (;;) {
@@ -1243,7 +1352,7 @@ static int http_read_header(URLContext *h, int *new_location)
 
         av_log(h, AV_LOG_TRACE, "header='%s'\n", line);
 
-        err = process_line(h, line, s->line_count, new_location);
+        err = process_line(h, line, s->line_count);
         if (err < 0)
             return err;
         if (err == 0)
@@ -1294,7 +1403,7 @@ static void bprint_escaped_path(AVBPrint *bp, const char *path)
 
 static int http_connect(URLContext *h, const char *path, const char *local_path,
                         const char *hoststr, const char *auth,
-                        const char *proxyauth, int *new_location)
+                        const char *proxyauth)
 {
     HTTPContext *s = h->priv_data;
     int post, err;
@@ -1438,11 +1547,11 @@ static int http_connect(URLContext *h, const char *path, const char *local_path,
     }
 
     /* wait for header */
-    err = http_read_header(h, new_location);
+    err = http_read_header(h);
     if (err < 0)
         goto done;
 
-    if (*new_location)
+    if (s->new_location)
         s->off = off;
 
     err = (off == s->off) ? 0 : -1;
@@ -1564,7 +1673,7 @@ static int64_t http_seek_internal(URLContext *h, int64_t off, int whence, int fo
 static int http_read_stream(URLContext *h, uint8_t *buf, int size)
 {
     HTTPContext *s = h->priv_data;
-    int err, new_location, read_ret;
+    int err, read_ret;
     int64_t seek_ret;
     int reconnect_delay = 0;
 
@@ -1572,7 +1681,7 @@ static int http_read_stream(URLContext *h, uint8_t *buf, int size)
         return AVERROR_EOF;
 
     if (s->end_chunked_post && !s->end_header) {
-        err = http_read_header(h, &new_location);
+        err = http_read_header(h);
         if (err < 0)
             return err;
     }
@@ -1785,6 +1894,8 @@ static int http_close(URLContext *h)
         ffurl_closep(&s->hd);
     av_dict_free(&s->chained_options);
     av_dict_free(&s->cookie_dict);
+    av_dict_free(&s->redirect_cache);
+    av_freep(&s->new_location);
     av_freep(&s->uri);
     return ret;
 }
@@ -1944,7 +2055,6 @@ static int http_proxy_open(URLContext *h, const char *uri, int flags)
     int port, ret = 0, attempts = 0;
     HTTPAuthType cur_auth_type;
     char *authstr;
-    int new_loc;
 
     if( s->seekable == 1 )
         h->is_streamed = 0;
@@ -1998,7 +2108,7 @@ redo:
      * since the client starts the conversation there, so there
      * is no extra data that we might buffer up here.
      */
-    ret = http_read_header(h, &new_loc);
+    ret = http_read_header(h);
     if (ret < 0)
         goto fail;