diff --git a/libavcodec/libopenjpegenc.c b/libavcodec/libopenjpegenc.c
index 3e52bcd4e9..08b7b6a152 100644
--- a/libavcodec/libopenjpegenc.c
+++ b/libavcodec/libopenjpegenc.c
@@ -345,7 +345,8 @@ static av_cold int libopenjpeg_encode_init(AVCodecContext *avctx)
     return 0;
 }
 
-static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame, opj_image_t *image)
+static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const uint8_t *src[4],
+                                    const int linesize[4], opj_image_t *image)
 {
     int compno;
     int x;
@@ -355,7 +356,7 @@ static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame,
     const int numcomps = image->numcomps;
 
     for (compno = 0; compno < numcomps; ++compno) {
-        if (image->comps[compno].w > frame->linesize[0] / numcomps) {
+        if (image->comps[compno].w > linesize[0] / numcomps) {
             av_log(avctx, AV_LOG_ERROR, "Error: frame's linesize is too small for the image\n");
             return 0;
         }
@@ -364,9 +365,9 @@ static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame,
     for (compno = 0; compno < numcomps; ++compno) {
         for (y = 0; y < avctx->height; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
-            frame_index = y * frame->linesize[0] + compno;
+            frame_index = y * linesize[0] + compno;
             for (x = 0; x < avctx->width; ++x) {
-                image_line[x] = frame->data[0][frame_index];
+                image_line[x] = src[0][frame_index];
                 frame_index += numcomps;
             }
             for (; x < image->comps[compno].w; ++x) {
@@ -385,17 +386,18 @@ static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame,
 }
 
 // for XYZ 12 bit
-static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const AVFrame *frame, opj_image_t *image)
+static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const uint8_t *src[4],
+                                    const int linesize[4], opj_image_t *image)
 {
     int compno;
     int x, y;
     int *image_line;
     int frame_index;
     const int numcomps  = image->numcomps;
-    uint16_t *frame_ptr = (uint16_t *)frame->data[0];
+    const uint16_t *frame_ptr = (const uint16_t *)src[0];
 
     for (compno = 0; compno < numcomps; ++compno) {
-        if (image->comps[compno].w > frame->linesize[0] / numcomps) {
+        if (image->comps[compno].w > linesize[0] / numcomps) {
             av_log(avctx, AV_LOG_ERROR, "Error: frame's linesize is too small for the image\n");
             return 0;
         }
@@ -404,7 +406,7 @@ static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const AVFrame *frame
     for (compno = 0; compno < numcomps; ++compno) {
         for (y = 0; y < avctx->height; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
-            frame_index = y * (frame->linesize[0] / 2) + compno;
+            frame_index = y * (linesize[0] / 2) + compno;
             for (x = 0; x < avctx->width; ++x) {
                 image_line[x] = frame_ptr[frame_index] >> 4;
                 frame_index += numcomps;
@@ -424,7 +426,8 @@ static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const AVFrame *frame
     return 1;
 }
 
-static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame, opj_image_t *image)
+static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const uint8_t *src[4],
+                                    const int linesize[4], opj_image_t *image)
 {
     int compno;
     int x;
@@ -432,10 +435,10 @@ static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame
     int *image_line;
     int frame_index;
     const int numcomps = image->numcomps;
-    uint16_t *frame_ptr = (uint16_t*)frame->data[0];
+    const uint16_t *frame_ptr = (const uint16_t*)src[0];
 
     for (compno = 0; compno < numcomps; ++compno) {
-        if (image->comps[compno].w > frame->linesize[0] / numcomps) {
+        if (image->comps[compno].w > linesize[0] / numcomps) {
             av_log(avctx, AV_LOG_ERROR, "Error: frame's linesize is too small for the image\n");
             return 0;
         }
@@ -444,7 +447,7 @@ static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame
     for (compno = 0; compno < numcomps; ++compno) {
         for (y = 0; y < avctx->height; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
-            frame_index = y * (frame->linesize[0] / 2) + compno;
+            frame_index = y * (linesize[0] / 2) + compno;
             for (x = 0; x < avctx->width; ++x) {
                 image_line[x] = frame_ptr[frame_index];
                 frame_index += numcomps;
@@ -464,7 +467,8 @@ static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame
     return 1;
 }
 
-static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *frame, opj_image_t *image)
+static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const uint8_t *src[4],
+                                    const int linesize[4], opj_image_t *image)
 {
     int compno;
     int x;
@@ -476,7 +480,7 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram
     const int numcomps = image->numcomps;
 
     for (compno = 0; compno < numcomps; ++compno) {
-        if (image->comps[compno].w > frame->linesize[compno]) {
+        if (image->comps[compno].w > linesize[compno]) {
             av_log(avctx, AV_LOG_ERROR, "Error: frame's linesize is too small for the image\n");
             return 0;
         }
@@ -487,9 +491,9 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram
         height = (avctx->height + image->comps[compno].dy - 1) / image->comps[compno].dy;
         for (y = 0; y < height; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
-            frame_index = y * frame->linesize[compno];
+            frame_index = y * linesize[compno];
             for (x = 0; x < width; ++x)
-                image_line[x] = frame->data[compno][frame_index++];
+                image_line[x] = src[compno][frame_index++];
             for (; x < image->comps[compno].w; ++x) {
                 image_line[x] = image_line[x - 1];
             }
@@ -505,7 +509,8 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram
     return 1;
 }
 
-static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *frame, opj_image_t *image)
+static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const uint8_t *src[4],
+                                    const int linesize[4], opj_image_t *image)
 {
     int compno;
     int x;
@@ -515,22 +520,21 @@ static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *fra
     int *image_line;
     int frame_index;
     const int numcomps = image->numcomps;
-    uint16_t *frame_ptr;
 
     for (compno = 0; compno < numcomps; ++compno) {
-        if (image->comps[compno].w > frame->linesize[compno]) {
+        if (image->comps[compno].w > linesize[compno]) {
             av_log(avctx, AV_LOG_ERROR, "Error: frame's linesize is too small for the image\n");
             return 0;
         }
     }
 
     for (compno = 0; compno < numcomps; ++compno) {
+        const uint16_t *frame_ptr = (const uint16_t *)src[compno];
         width     = (avctx->width + image->comps[compno].dx - 1) / image->comps[compno].dx;
         height    = (avctx->height + image->comps[compno].dy - 1) / image->comps[compno].dy;
-        frame_ptr = (uint16_t *)frame->data[compno];
         for (y = 0; y < height; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
-            frame_index = y * (frame->linesize[compno] / 2);
+            frame_index = y * (linesize[compno] / 2);
             for (x = 0; x < width; ++x)
                 image_line[x] = frame_ptr[frame_index++];
             for (; x < image->comps[compno].w; ++x) {
@@ -553,12 +557,15 @@ static int libopenjpeg_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
 {
     LibOpenJPEGContext *ctx = avctx->priv_data;
     int ret;
-    AVFrame *gbrframe;
     int cpyresult = 0;
     PacketWriter writer     = { 0 };
     opj_codec_t *compress   = NULL;
     opj_stream_t *stream    = NULL;
     opj_image_t *image      = mj2_create_image(avctx, &ctx->enc_params);
+    const uint8_t *data[4] = { frame->data[0], frame->data[1],
+                               frame->data[2], frame->data[3] };
+    int linesize[4]        = { frame->linesize[0], frame->linesize[1],
+                               frame->linesize[2], frame->linesize[3] };
     if (!image) {
         av_log(avctx, AV_LOG_ERROR, "Error creating the mj2 image\n");
         ret = AVERROR(EINVAL);
@@ -569,15 +576,15 @@ static int libopenjpeg_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     case AV_PIX_FMT_RGB24:
     case AV_PIX_FMT_RGBA:
     case AV_PIX_FMT_YA8:
-        cpyresult = libopenjpeg_copy_packed8(avctx, frame, image);
+        cpyresult = libopenjpeg_copy_packed8(avctx, data, linesize, image);
         break;
     case AV_PIX_FMT_XYZ12:
-        cpyresult = libopenjpeg_copy_packed12(avctx, frame, image);
+        cpyresult = libopenjpeg_copy_packed12(avctx, data, linesize, image);
         break;
     case AV_PIX_FMT_RGB48:
     case AV_PIX_FMT_RGBA64:
     case AV_PIX_FMT_YA16:
-        cpyresult = libopenjpeg_copy_packed16(avctx, frame, image);
+        cpyresult = libopenjpeg_copy_packed16(avctx, data, linesize, image);
         break;
     case AV_PIX_FMT_GBR24P:
     case AV_PIX_FMT_GBRP9:
@@ -585,23 +592,17 @@ static int libopenjpeg_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     case AV_PIX_FMT_GBRP12:
     case AV_PIX_FMT_GBRP14:
     case AV_PIX_FMT_GBRP16:
-        gbrframe = av_frame_clone(frame);
-        if (!gbrframe) {
-            ret = AVERROR(ENOMEM);
-            goto done;
-        }
-        gbrframe->data[0] = frame->data[2]; // swap to be rgb
-        gbrframe->data[1] = frame->data[0];
-        gbrframe->data[2] = frame->data[1];
-        gbrframe->linesize[0] = frame->linesize[2];
-        gbrframe->linesize[1] = frame->linesize[0];
-        gbrframe->linesize[2] = frame->linesize[1];
+        data[0] = frame->data[2]; // swap to be rgb
+        data[1] = frame->data[0];
+        data[2] = frame->data[1];
+        linesize[0] = frame->linesize[2];
+        linesize[1] = frame->linesize[0];
+        linesize[2] = frame->linesize[1];
         if (avctx->pix_fmt == AV_PIX_FMT_GBR24P) {
-            cpyresult = libopenjpeg_copy_unpacked8(avctx, gbrframe, image);
+            cpyresult = libopenjpeg_copy_unpacked8(avctx, data, linesize, image);
         } else {
-            cpyresult = libopenjpeg_copy_unpacked16(avctx, gbrframe, image);
+            cpyresult = libopenjpeg_copy_unpacked16(avctx, data, linesize, image);
         }
-        av_frame_free(&gbrframe);
         break;
     case AV_PIX_FMT_GRAY8:
     case AV_PIX_FMT_YUV410P:
@@ -613,7 +614,7 @@ static int libopenjpeg_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     case AV_PIX_FMT_YUVA420P:
     case AV_PIX_FMT_YUVA422P:
     case AV_PIX_FMT_YUVA444P:
-        cpyresult = libopenjpeg_copy_unpacked8(avctx, frame, image);
+        cpyresult = libopenjpeg_copy_unpacked8(avctx, data, linesize, image);
         break;
     case AV_PIX_FMT_GRAY10:
     case AV_PIX_FMT_GRAY12:
@@ -643,7 +644,7 @@ static int libopenjpeg_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     case AV_PIX_FMT_YUVA444P16:
     case AV_PIX_FMT_YUVA422P16:
     case AV_PIX_FMT_YUVA420P16:
-        cpyresult = libopenjpeg_copy_unpacked16(avctx, frame, image);
+        cpyresult = libopenjpeg_copy_unpacked16(avctx, data, linesize, image);
         break;
     default:
         av_log(avctx, AV_LOG_ERROR,