diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 38f5a2c874..6af64ba586 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -12,9 +12,13 @@ OBJS = allfilters.o \
 HEADERS = avfilter.h
 
 EXTRALIBS := -L$(BUILD_ROOT)/libavcodec -lavcodec$(BUILDSUF) \
-             -L$(BUILD_ROOT)/libswscale -lswscale$(BUILDSUF) \
              -L$(BUILD_ROOT)/libavutil  -lavutil$(BUILDSUF) $(EXTRALIBS)
 
+ifeq ($(CONFIG_SWSCALER),yes)
+EXTRALIBS+=-L$(BUILD_ROOT)/libswscale
+EXTRALIBS+=-lswscale$(BUILDSUF)
+endif
+
 ifeq ($(CONFIG_AVFILTER_LAVF),yes)
 EXTRALIBS := -L$(BUILD_ROOT)/libavformat -lavformat$(BUILDSUF) $(EXTRALIBS)
 endif