diff --git a/libavutil/hwcontext_dxva2.c b/libavutil/hwcontext_dxva2.c
index 89fedd2cd5..4ed0d56aea 100644
--- a/libavutil/hwcontext_dxva2.c
+++ b/libavutil/hwcontext_dxva2.c
@@ -331,6 +331,9 @@ static int dxva2_transfer_data_to(AVHWFramesContext *ctx, AVFrame *dst,
     AVFrame *map;
     int ret;
 
+    if (src->format != ctx->sw_format)
+        return AVERROR(ENOSYS);
+
     map = av_frame_alloc();
     if (!map)
         return AVERROR(ENOMEM);
@@ -355,6 +358,9 @@ static int dxva2_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
     ptrdiff_t src_linesize[4], dst_linesize[4];
     int ret, i;
 
+    if (dst->format != ctx->sw_format)
+        return AVERROR(ENOSYS);
+
     map = av_frame_alloc();
     if (!map)
         return AVERROR(ENOMEM);