diff --git a/libavcodec/ansi.c b/libavcodec/ansi.c
index c2a1d83974..861d4e57e4 100644
--- a/libavcodec/ansi.c
+++ b/libavcodec/ansi.c
@@ -325,6 +325,11 @@ static int decode_frame(AVCodecContext *avctx,
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
         return ret;
     }
+    if (!avctx->frame_number) {
+        memset(s->frame.data[0], 0, avctx->height * FFABS(s->frame.linesize[0]));
+        memset(s->frame.data[1], 0, AVPALETTE_SIZE);
+    }
+
     s->frame.pict_type           = AV_PICTURE_TYPE_I;
     s->frame.palette_has_changed = 1;
     memcpy(s->frame.data[1], ff_cga_palette, 16 * 4);
diff --git a/tests/ref/fate/ansi b/tests/ref/fate/ansi
index 56c0226095..78e853fc0b 100644
--- a/tests/ref/fate/ansi
+++ b/tests/ref/fate/ansi
@@ -1,7 +1,7 @@
 #tb 0: 1/25
-0,          0,          0,        1,   768000, 0x3032d0de
-0,          1,          1,        1,   768000, 0xc3be5922
-0,          2,          2,        1,   768000, 0xf530c476
+0,          0,          0,        1,   768000, 0x772dd3d0
+0,          1,          1,        1,   768000, 0xd7dab1d1
+0,          2,          2,        1,   768000, 0x0e56f2d3
 0,          3,          3,        1,   768000, 0x11c1fb8e
 0,          4,          4,        1,   768000, 0x72d12da9
 0,          5,          5,        1,   768000, 0x39c7a70d