RepoMirrors
/
openwrt
mirror of git://git.openwrt.org/openwrt/openwrt.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

build: make find_md5 reproducible with AUTOREMOVE 

While experimenting with the AUTOREMOVE option in search of a way to use
prebuilt host tools in different buildroot, it was discovered that the
md5 generated by find_md5 in depends.mk is not reproducible.

Currently the hash is generated by the path of the file in addition to
the file mod time. Out of confusion, probably, there was an idea that
such command was used on the package build_dir. Reality is that this
command is run on the package files. (Makefile, patches, src)

This is problematic because the package Makefile (for example) change at
each git clone and base the hash on the Makefile mtime doesn't really
reflect if the Makefile actually changes across a buildroot or not.

A better approach is to generate an hash of each file and then generate
an hash on the sort hash list. This way we remove the problem of git
clone setting a wrong mtime while keeping the integrity of checking if a
file changed for the package as any change will result in a different
hash.

Introduce a new kind of find_md5 function, find_md5_reproducible that
apply this new logic and limit it only with AUTOREMOVE option set to
prevent any kind of slowdown due to additional hash generation.

Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>
This commit is contained in:
Christian Marangi 2022-09-06 16:48:37 +02:00
parent 0e3d51ccae
commit 53a08e3743
No known key found for this signature in database
GPG Key ID: AC001D09ADBFEAD7
4 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/depends.mk
View File

@ -12,6 +12,7 @@
DEP_FINDPARAMS := -x "*/.svn*" -x ".*" -x "*:*" -x "*\!*" -x "* *" -x "*\\\#*" -x "*/.*_check" -x "*/.*.swp" -x "*/.pkgdir*" DEP_FINDPARAMS := -x "*/.svn*" -x ".*" -x "*:*" -x "*\!*" -x "* *" -x "*\\\#*" -x "*/.*_check" -x "*/.*.swp" -x "*/.pkgdir*"
find_md5=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -printf "%p%T@\n" | sort | $(MKHASH) md5 find_md5=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -printf "%p%T@\n" | sort | $(MKHASH) md5
find_md5_reproducible=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) -print0 | xargs -0 $(MKHASH) md5 | sort | $(MKHASH) md5
define rdep define rdep
.PRECIOUS: $(2) .PRECIOUS: $(2)

2
include/host-build.mk
View File

@ -21,7 +21,7 @@ include $(INCLUDE_DIR)/depends.mk
include $(INCLUDE_DIR)/quilt.mk include $(INCLUDE_DIR)/quilt.mk
BUILD_TYPES += host BUILD_TYPES += host
HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS))) HOST_STAMP_PREPARED=$(HOST_BUILD_DIR)/.prepared$(if $(HOST_QUILT)$(DUMP),,$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(HOST_PREPARED_DEPENDS)))
HOST_STAMP_CONFIGURED:=$(HOST_BUILD_DIR)/.configured HOST_STAMP_CONFIGURED:=$(HOST_BUILD_DIR)/.configured
HOST_STAMP_BUILT:=$(HOST_BUILD_DIR)/.built HOST_STAMP_BUILT:=$(HOST_BUILD_DIR)/.built
HOST_BUILD_PREFIX?=$(if $(IS_PACKAGE_BUILD),$(STAGING_DIR_HOSTPKG),$(STAGING_DIR_HOST)) HOST_BUILD_PREFIX?=$(if $(IS_PACKAGE_BUILD),$(STAGING_DIR_HOSTPKG),$(STAGING_DIR_HOST))

2
include/kernel-build.mk
View File

@ -10,7 +10,7 @@ ifneq ($(DUMP),1)
endif endif
KERNEL_FILE_DEPENDS=$(GENERIC_BACKPORT_DIR) $(GENERIC_PATCH_DIR) $(GENERIC_HACK_DIR) $(PATCH_DIR) $(GENERIC_FILES_DIR) $(FILES_DIR) KERNEL_FILE_DEPENDS=$(GENERIC_BACKPORT_DIR) $(GENERIC_PATCH_DIR) $(GENERIC_HACK_DIR) $(PATCH_DIR) $(GENERIC_FILES_DIR) $(FILES_DIR)
STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,$(KERNEL_FILE_DEPENDS),))) STAMP_PREPARED=$(LINUX_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),$(KERNEL_FILE_DEPENDS),)))
STAMP_CONFIGURED:=$(LINUX_DIR)/.configured STAMP_CONFIGURED:=$(LINUX_DIR)/.configured
include $(INCLUDE_DIR)/download.mk include $(INCLUDE_DIR)/download.mk
include $(INCLUDE_DIR)/quilt.mk include $(INCLUDE_DIR)/quilt.mk

2
include/package.mk
View File

@ -85,7 +85,7 @@ ifneq ($(PREV_STAMP_PREPARED),)
STAMP_PREPARED:=$(PREV_STAMP_PREPARED) STAMP_PREPARED:=$(PREV_STAMP_PREPARED)
CONFIG_AUTOREBUILD:= CONFIG_AUTOREBUILD:=
else else
STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call find_md5,${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS))) STAMP_PREPARED=$(PKG_BUILD_DIR)/.prepared$(if $(QUILT)$(DUMP),,_$(shell $(call $(if $(CONFIG_AUTOREMOVE),find_md5_reproducible,find_md5),${CURDIR} $(PKG_FILE_DEPENDS),))_$(call confvar,CONFIG_AUTOREMOVE $(PKG_PREPARED_DEPENDS)))
endif endif
STAMP_CONFIGURED=$(PKG_BUILD_DIR)/.configured$(if $(DUMP),,_$(call confvar,$(PKG_CONFIG_DEPENDS))) STAMP_CONFIGURED=$(PKG_BUILD_DIR)/.configured$(if $(DUMP),,_$(call confvar,$(PKG_CONFIG_DEPENDS)))
STAMP_CONFIGURED_WILDCARD=$(PKG_BUILD_DIR)/.configured_* STAMP_CONFIGURED_WILDCARD=$(PKG_BUILD_DIR)/.configured_*