diff --git a/libavfilter/avfilter.c b/libavfilter/avfilter.c
index 0518e6d6c2..72e0a87f8e 100644
--- a/libavfilter/avfilter.c
+++ b/libavfilter/avfilter.c
@@ -73,15 +73,10 @@ AVFilterBufferRef *avfilter_ref_buffer(AVFilterBufferRef *ref, int pmask)
 static void store_in_pool(AVFilterBufferRef *ref)
 {
     int i;
-    AVFilterLink *link= ref->buf->priv;
-    AVFilterPool *pool;
+    AVFilterPool *pool= ref->buf->priv;
 
     av_assert0(ref->buf->data[0]);
 
-    if(!link->pool)
-        link->pool = av_mallocz(sizeof(AVFilterPool));
-    pool= link->pool;
-
     if(pool->count == POOL_SIZE){
         AVFilterBufferRef *ref1= pool->pic[0];
         av_freep(&ref1->video);
@@ -685,7 +680,6 @@ void avfilter_free(AVFilterContext *filter)
         if ((link = filter->inputs[i])) {
             if (link->src)
                 link->src->outputs[link->srcpad - link->src->output_pads] = NULL;
-            av_freep(&link->pool);
             avfilter_formats_unref(&link->in_formats);
             avfilter_formats_unref(&link->out_formats);
         }
@@ -695,7 +689,6 @@ void avfilter_free(AVFilterContext *filter)
         if ((link = filter->outputs[i])) {
             if (link->dst)
                 link->dst->inputs[link->dstpad - link->dst->input_pads] = NULL;
-            av_freep(&link->pool);
             avfilter_formats_unref(&link->in_formats);
             avfilter_formats_unref(&link->out_formats);
         }
diff --git a/libavfilter/defaults.c b/libavfilter/defaults.c
index 63bca6b51c..9ee23e57b7 100644
--- a/libavfilter/defaults.c
+++ b/libavfilter/defaults.c
@@ -57,7 +57,8 @@ AVFilterBufferRef *avfilter_default_get_video_buffer(AVFilterLink *link, int per
             memcpy(picref->linesize, pic->linesize, sizeof(picref->linesize));
             return picref;
         }
-    }
+    }else
+        pool = link->pool = av_mallocz(sizeof(AVFilterPool));
 
     // +2 is needed for swscaler, +16 to be SIMD-friendly
     if ((i=av_image_alloc(data, linesize, w, h, link->format, 16)) < 0)
@@ -71,7 +72,7 @@ AVFilterBufferRef *avfilter_default_get_video_buffer(AVFilterLink *link, int per
     }
     memset(data[0], 128, i);
 
-    picref->buf->priv= link;
+    picref->buf->priv= pool;
     picref->buf->free= NULL;
 
     return picref;