diff --git a/libswscale/aarch64/swscale.c b/libswscale/aarch64/swscale.c
index 54a3beabe8..eecbea88ca 100644
--- a/libswscale/aarch64/swscale.c
+++ b/libswscale/aarch64/swscale.c
@@ -34,7 +34,10 @@ av_cold void ff_sws_init_swscale_aarch64(SwsContext *c)
     int cpu_flags = av_get_cpu_flags();
 
     if (have_neon(cpu_flags)) {
-        if (c->srcBpc == 8 && c->dstBpc <= 14) {
+        if (c->srcBpc == 8 && c->dstBpc <= 14 &&
+            (c->hLumFilterSize % 8) == 0 &&
+            (c->hChrFilterSize % 8) == 0)
+        {
             c->hyScale = c->hcScale = ff_hscale_8_to_15_neon;
         }
         if (c->dstBpc == 8) {