diff --git a/libavcodec/ivi.c b/libavcodec/ivi.c
index 73fcf51b7b..18192cbf23 100644
--- a/libavcodec/ivi.c
+++ b/libavcodec/ivi.c
@@ -354,23 +354,11 @@ av_cold int ff_ivi_init_planes(AVCodecContext *avctx, IVIPlaneDesc *planes, cons
             band->height   = b_height;
             band->pitch    = width_aligned;
             band->aheight  = height_aligned;
-            band->bufs[0]  = av_mallocz(buf_size);
-            band->bufs[1]  = av_mallocz(buf_size);
+            av_assert0(!band->bufs[0] && !band->bufs[1] &&
+                       !band->bufs[2] && !band->bufs[3]);
             band->bufsize  = buf_size/2;
-            if (!band->bufs[0] || !band->bufs[1])
-                return AVERROR(ENOMEM);
+            av_assert0(buf_size % 2 == 0);
 
-            /* allocate the 3rd band buffer for scalability mode */
-            if (cfg->luma_bands > 1) {
-                band->bufs[2] = av_mallocz(buf_size);
-                if (!band->bufs[2])
-                    return AVERROR(ENOMEM);
-            }
-            if (is_indeo4) {
-                band->bufs[3]  = av_mallocz(buf_size);
-                if (!band->bufs[3])
-                    return AVERROR(ENOMEM);
-            }
             /* reset custom vlc */
             planes[p].bands[0].blk_vlc.cust_desc.num_rows = 0;
         }
@@ -945,6 +933,15 @@ static void ivi_output_plane(IVIPlaneDesc *plane, uint8_t *dst, ptrdiff_t dst_pi
     }
 }
 
+static void *prepare_buf(IVI45DecContext *ctx, IVIBandDesc *band, int i)
+{
+    if (ctx->pic_conf.luma_bands <= 1 && i == 2)
+        return NULL;
+    if (!band->bufs[i])
+        band->bufs[i] = av_mallocz(2 * band->bufsize);
+    return band->bufs[i];
+}
+
 /**
  *  Decode an Indeo 4 or 5 band.
  *
@@ -959,18 +956,22 @@ static int decode_band(IVI45DecContext *ctx,
     int         result, i, t, idx1, idx2, pos;
     IVITile     *tile;
 
-    band->buf     = band->bufs[ctx->dst_buf];
+    band->buf     = prepare_buf(ctx, band, ctx->dst_buf);
     if (!band->buf) {
         av_log(avctx, AV_LOG_ERROR, "Band buffer points to no data!\n");
         return AVERROR_INVALIDDATA;
     }
     if (ctx->is_indeo4 && ctx->frame_type == IVI4_FRAMETYPE_BIDIR) {
-        band->ref_buf   = band->bufs[ctx->b_ref_buf];
-        band->b_ref_buf = band->bufs[ctx->ref_buf];
+        band->ref_buf   = prepare_buf(ctx, band, ctx->b_ref_buf);
+        band->b_ref_buf = prepare_buf(ctx, band, ctx->ref_buf);
+        if (!band->b_ref_buf)
+            return AVERROR(ENOMEM);
     } else {
-        band->ref_buf   = band->bufs[ctx->ref_buf];
+        band->ref_buf   = prepare_buf(ctx, band, ctx->ref_buf);
         band->b_ref_buf = 0;
     }
+    if (!band->ref_buf)
+        return AVERROR(ENOMEM);
     band->data_ptr  = ctx->frame_data + (get_bits_count(&ctx->gb) >> 3);
 
     result = ctx->decode_band_hdr(ctx, band, avctx);