diff --git a/kpatch-build/Makefile b/kpatch-build/Makefile
index 0db5643..7505e07 100644
--- a/kpatch-build/Makefile
+++ b/kpatch-build/Makefile
@@ -13,12 +13,14 @@ SOURCES = create-diff-object.c kpatch-elf.c \
 ifeq ($(ARCH),x86_64)
 SOURCES += insn/insn.c insn/inat.c
 INSN     = insn/insn.o insn/inat.o
+insn/%.o: CFLAGS := $(filter-out -Wconversion, $(CFLAGS))
 else ifeq ($(ARCH),ppc64le)
 SOURCES += gcc-plugins/ppc64le-plugin.c
 PLUGIN   = gcc-plugins/ppc64le-plugin.so
 TARGETS += $(PLUGIN)
 GCC_PLUGINS_DIR := $(shell gcc -print-file-name=plugin)
-PLUGIN_CFLAGS    = -shared $(CFLAGS) -I$(GCC_PLUGINS_DIR)/include \
+PLUGIN_CFLAGS := $(filter-out -Wconversion, $(CFLAGS))
+PLUGIN_CFLAGS += -shared -I$(GCC_PLUGINS_DIR)/include \
 		   -Igcc-plugins -fPIC -fno-rtti -O2 -Wall
 endif