diff --git a/libavfilter/drawutils.c b/libavfilter/drawutils.c
index 6b46f5803d..0965afb03e 100644
--- a/libavfilter/drawutils.c
+++ b/libavfilter/drawutils.c
@@ -105,6 +105,9 @@ int ff_draw_init(FFDrawContext *draw, enum AVPixelFormat format, unsigned flags)
             return AVERROR(ENOSYS);
         if (c->plane >= MAX_PLANES)
             return AVERROR(ENOSYS);
+        /* data must either be in the high or low bits, never middle */
+        if (c->shift && ((c->shift + c->depth) & 0x7))
+            return AVERROR(ENOSYS);
         /* strange interleaving */
         if (pixelstep[c->plane] != 0 &&
             pixelstep[c->plane] != c->step)
diff --git a/tests/ref/fate/filter-pixfmts-pad b/tests/ref/fate/filter-pixfmts-pad
index 1362d198b4..74981cd6c1 100644
--- a/tests/ref/fate/filter-pixfmts-pad
+++ b/tests/ref/fate/filter-pixfmts-pad
@@ -32,8 +32,6 @@ p416le              6db094f8d7d27d7299bf9496ad66e2e0
 rgb0                78d500c8361ab6423a4826a00268c908
 rgb24               17f9e2e0c609009acaf2175c42d4a2a5
 rgba                b157c90191463d34fb3ce77b36c96386
-x2bgr10le           d4aff89f5e15ccbb1812f319874ed444
-x2rgb10le           a0c5925bd56b6f85f918c4e9fb93e90e
 xyz12le             85abf80b77a9236a76ba0b00fcbdea2d
 ya16le              940fafa240b9916de5f73cb20a552f24
 ya8                 5fc0f471207ddf7aa01b07027d56b672