diff --git a/libavcodec/h261dec.c b/libavcodec/h261dec.c
index c41b96c3c7..6cdf11f822 100644
--- a/libavcodec/h261dec.c
+++ b/libavcodec/h261dec.c
@@ -94,7 +94,6 @@ static av_cold int h261_decode_init(AVCodecContext *avctx)
     avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 
     h->gob_start_code_skipped = 0;
-    ff_mpv_idct_init(s);
 
     ff_thread_once(&init_static_once, h261_decode_init_static);
 
diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c
index 9f63f1a7cb..544d32b682 100644
--- a/libavcodec/h263dec.c
+++ b/libavcodec/h263dec.c
@@ -128,7 +128,6 @@ av_cold int ff_h263_decode_init(AVCodecContext *avctx)
         avctx->codec->id != AV_CODEC_ID_H263P &&
         avctx->codec->id != AV_CODEC_ID_MPEG4) {
         avctx->pix_fmt = h263_get_format(avctx);
-        ff_mpv_idct_init(s);
         if ((ret = ff_mpv_common_init(s)) < 0)
             return ret;
     }
@@ -459,10 +458,6 @@ retry:
     if (ret < 0)
         return ret;
 
-    if (!s->context_initialized)
-        // we need the idct permutation for reading a custom matrix
-        ff_mpv_idct_init(s);
-
     /* let's go :-) */
     if (CONFIG_WMV2_DECODER && s->msmpeg4_version == 5) {
         ret = ff_wmv2_decode_picture_header(s);
diff --git a/libavcodec/mpeg12dec.c b/libavcodec/mpeg12dec.c
index d2dbcd5b61..c0f1e8763b 100644
--- a/libavcodec/mpeg12dec.c
+++ b/libavcodec/mpeg12dec.c
@@ -1057,7 +1057,6 @@ static av_cold int mpeg_decode_init(AVCodecContext *avctx)
         avctx->coded_width = avctx->coded_height = 0; // do not trust dimensions from input
     ff_mpv_decode_init(s2, avctx);
 
-    ff_mpv_idct_init(s2);
     ff_mpeg12_init_vlcs();
 
     s2->chroma_format              = 1;
@@ -3048,7 +3047,6 @@ static av_cold int ipu_decode_init(AVCodecContext *avctx)
     avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 
     ff_mpv_decode_init(m, avctx);
-    ff_mpv_idct_init(m);
     ff_mpeg12_init_vlcs();
 
     for (int i = 0; i < 64; i++) {
diff --git a/libavcodec/mpegvideo_dec.c b/libavcodec/mpegvideo_dec.c
index 452650b487..b4ecb56987 100644
--- a/libavcodec/mpegvideo_dec.c
+++ b/libavcodec/mpegvideo_dec.c
@@ -54,6 +54,7 @@ void ff_mpv_decode_init(MpegEncContext *s, AVCodecContext *avctx)
     /* convert fourcc to upper case */
     s->codec_tag       = ff_toupper4(avctx->codec_tag);
 
+    ff_mpv_idct_init(s);
     ff_h264chroma_init(&s->h264chroma, 8); //for lowres
 }
 
@@ -84,7 +85,6 @@ int ff_mpeg_update_thread_context(AVCodecContext *dst,
         s->bitstream_buffer_size = s->allocated_bitstream_buffer_size = 0;
 
         if (s1->context_initialized) {
-            ff_mpv_idct_init(s);
             if ((err = ff_mpv_common_init(s)) < 0)
                 return err;
         }
diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c
index 6abceade4e..fbecfdae9f 100644
--- a/libavcodec/rv10.c
+++ b/libavcodec/rv10.c
@@ -420,7 +420,6 @@ static av_cold int rv10_decode_init(AVCodecContext *avctx)
 
     avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 
-    ff_mpv_idct_init(s);
     if ((ret = ff_mpv_common_init(s)) < 0)
         return ret;
 
diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c
index e9660bb457..152c62fa07 100644
--- a/libavcodec/rv34.c
+++ b/libavcodec/rv34.c
@@ -1498,7 +1498,6 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx)
     avctx->has_b_frames = 1;
     s->low_delay = 0;
 
-    ff_mpv_idct_init(s);
     if ((ret = ff_mpv_common_init(s)) < 0)
         return ret;
 
diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c
index 449d2fea5e..a4f2d30e22 100644
--- a/libavcodec/vc1dec.c
+++ b/libavcodec/vc1dec.c
@@ -431,7 +431,6 @@ av_cold int ff_vc1_decode_init(AVCodecContext *avctx)
         return ret;
 
     ff_mpv_decode_init(s, avctx);
-    ff_mpv_idct_init(s);
 
     avctx->pix_fmt = vc1_get_format(avctx);