diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..100bbceb --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# To use this config on you editor, follow the instructions at: +# http://editorconfig.org + +root = true + +[*] +charset = utf-8 +insert_final_newline = true +tab_width = 8 +max_line_length = 90 + +[Makefile] +indent_style = tab + +[*.{sh,py}] +indent_style = space +indent_size = 4 + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae96f7d8..e05ef3bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ check-commits: stage: test image: registry.gitlab.com/kernel-firmware/linux-firmware rules: - - if: $CI_MERGE_REQUEST_ID + - if: $CI_MERGE_REQUEST_ID script: - ci-fairy check-commits --signed-off-by --textwidth=0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a49d063..370a2970 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,30 +1,30 @@ default_stages: [commit] repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: - - id: check-executables-have-shebangs - - id: forbid-new-submodules - - id: check-yaml - - id: check-symlinks - - id: destroyed-symlinks -- repo: https://github.com/shellcheck-py/shellcheck-py + - id: check-executables-have-shebangs + - id: forbid-new-submodules + - id: check-yaml + - id: check-symlinks + - id: destroyed-symlinks + - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.9.0.5 hooks: - - id: shellcheck -- repo: https://github.com/ambv/black + - id: shellcheck + - repo: https://github.com/ambv/black rev: 22.6.0 hooks: - - id: black -- repo: https://github.com/igorshubovych/markdownlint-cli + - id: black + - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.33.0 hooks: - - id: markdownlint - args: ['--fix'] -- repo: local + - id: markdownlint + args: ['--fix'] + - repo: local hooks: - - id: check-whence - name: Check whence - files: 'WHENCE' - language: script - entry: ./check_whence.py + - id: check-whence + name: Check whence + files: 'WHENCE' + language: script + entry: ./check_whence.py diff --git a/Makefile b/Makefile index ac43c97f..216dd488 100644 --- a/Makefile +++ b/Makefile @@ -26,21 +26,22 @@ deb: rpm: ./build_packages.py --rpm -install: - install -d $(DESTDIR)$(FIRMWAREDIR) - ./copy-firmware.sh $(COPYOPTS) $(DESTDIR)$(FIRMWAREDIR) +install: install-nodedup + ./dedup-firmware.sh $(DESTDIR)$(FIRMWAREDIR) install-nodedup: install -d $(DESTDIR)$(FIRMWAREDIR) - ./copy-firmware.sh --ignore-duplicates $(DESTDIR)$(FIRMWAREDIR) + ./copy-firmware.sh $(DESTDIR)$(FIRMWAREDIR) install-xz: install -d $(DESTDIR)$(FIRMWAREDIR) ./copy-firmware.sh --xz $(DESTDIR)$(FIRMWAREDIR) + ./dedup-firmware.sh $(DESTDIR)$(FIRMWAREDIR) install-zst: install -d $(DESTDIR)$(FIRMWAREDIR) ./copy-firmware.sh --zstd $(DESTDIR)$(FIRMWAREDIR) + ./dedup-firmware.sh $(DESTDIR)$(FIRMWAREDIR) clean: rm -rf release dist diff --git a/WHENCE b/WHENCE index e3e085b8..9678e670 100644 --- a/WHENCE +++ b/WHENCE @@ -3699,7 +3699,7 @@ File: ath10k/WCN3990/hw1.0/qcm2290/wlanmdsp.mbn Version: WLAN.HL.3.3.7.c2-00931-QCAHLSWMTPLZ-1 Link: ath10k/WCN3990/hw1.0/qrb4210/wlanmdsp.mbn -> ../qcm2290/wlanmdsp.mbn Link: qcom/qcm2290/wlanmdsp.mbn -> ../../ath10k/WCN3990/hw1.0/qcm2290/wlanmdsp.mbn -Link: qcom/qrb4210/wlanmdsp.mbn -> ../../ath10k/WCN3990/hw1.0/qrb4210/wlanmdsp.mbn +Link: qcom/qrb4210/wlanmdsp.mbn -> ../../ath10k/WCN3990/hw1.0/qcm2290/wlanmdsp.mbn File: ath10k/WCN3990/hw1.0/qcm2290/firmware-5.bin Link: ath10k/WCN3990/hw1.0/qrb4210/firmware-5.bin -> ../qcm2290/firmware-5.bin @@ -5227,11 +5227,11 @@ Link: nvidia/gp104/acr/bl.bin -> ../../gp102/acr/bl.bin Link: nvidia/gp104/acr/ucode_load.bin -> ../../gp102/acr/ucode_load.bin Link: nvidia/gp104/acr/ucode_unload.bin -> ../../gp102/acr/ucode_unload.bin Link: nvidia/gp104/acr/unload_bl.bin -> ../../gp102/acr/unload_bl.bin -Link: nvidia/gp104/gr/fecs_bl.bin -> ../../gp102/gr/fecs_bl.bin +Link: nvidia/gp104/gr/fecs_bl.bin -> ../../gm200/gr/fecs_bl.bin File: nvidia/gp104/gr/fecs_data.bin File: nvidia/gp104/gr/fecs_inst.bin File: nvidia/gp104/gr/fecs_sig.bin -Link: nvidia/gp104/gr/gpccs_bl.bin -> ../../gp102/gr/gpccs_bl.bin +Link: nvidia/gp104/gr/gpccs_bl.bin -> ../../gm200/gr/gpccs_bl.bin File: nvidia/gp104/gr/gpccs_data.bin File: nvidia/gp104/gr/gpccs_inst.bin File: nvidia/gp104/gr/gpccs_sig.bin @@ -5250,11 +5250,11 @@ Link: nvidia/gp106/acr/bl.bin -> ../../gp102/acr/bl.bin Link: nvidia/gp106/acr/ucode_load.bin -> ../../gp102/acr/ucode_load.bin Link: nvidia/gp106/acr/ucode_unload.bin -> ../../gp102/acr/ucode_unload.bin Link: nvidia/gp106/acr/unload_bl.bin -> ../../gp102/acr/unload_bl.bin -Link: nvidia/gp106/gr/fecs_bl.bin -> ../../gp102/gr/fecs_bl.bin +Link: nvidia/gp106/gr/fecs_bl.bin -> ../../gm200/gr/fecs_bl.bin File: nvidia/gp106/gr/fecs_data.bin Link: nvidia/gp106/gr/fecs_inst.bin -> ../../gp102/gr/fecs_inst.bin File: nvidia/gp106/gr/fecs_sig.bin -Link: nvidia/gp106/gr/gpccs_bl.bin -> ../../gp102/gr/gpccs_bl.bin +Link: nvidia/gp106/gr/gpccs_bl.bin -> ../../gm200/gr/gpccs_bl.bin File: nvidia/gp106/gr/gpccs_data.bin Link: nvidia/gp106/gr/gpccs_inst.bin -> ../../gp102/gr/gpccs_inst.bin File: nvidia/gp106/gr/gpccs_sig.bin @@ -5998,10 +5998,10 @@ File: netronome/flower/nic_AMDA0096.nffw File: netronome/flower/nic_AMDA0097.nffw File: netronome/flower/nic_AMDA0058.nffw Link: netronome/flower/nic_AMDA0081.nffw -> nic_AMDA0097.nffw -Link: netronome/flower/nic_AMDA0081-0001_1x40.nffw -> nic_AMDA0081.nffw +Link: netronome/flower/nic_AMDA0081-0001_1x40.nffw -> nic_AMDA0097.nffw Link: netronome/flower/nic_AMDA0097-0001_2x40.nffw -> nic_AMDA0097.nffw Link: netronome/flower/nic_AMDA0099-0001_2x10.nffw -> nic_AMDA0099.nffw -Link: netronome/flower/nic_AMDA0081-0001_4x10.nffw -> nic_AMDA0081.nffw +Link: netronome/flower/nic_AMDA0081-0001_4x10.nffw -> nic_AMDA0097.nffw Link: netronome/flower/nic_AMDA0097-0001_4x10_1x40.nffw -> nic_AMDA0097.nffw Link: netronome/flower/nic_AMDA0099-0001_2x25.nffw -> nic_AMDA0099.nffw Link: netronome/flower/nic_AMDA0096-0001_2x10.nffw -> nic_AMDA0096.nffw diff --git a/check_whence.py b/check_whence.py index fd74a56a..09dbdd54 100755 --- a/check_whence.py +++ b/check_whence.py @@ -37,7 +37,7 @@ def list_whence_files(): for line in whence: match = re.match(r"(?:RawFile|File):\s*(.*)", line) if match: - yield match.group(1).replace("\ ", " ").replace('"', "") + yield match.group(1).replace(r"\ ", " ").replace('"', "") continue @@ -48,8 +48,8 @@ def list_links_list(): if match: linkname, target = match.group(1).split("->") - linkname = linkname.strip().replace("\ ", " ").replace('"', "") - target = target.strip().replace("\ ", " ").replace('"', "") + linkname = linkname.strip().replace(r"\ ", " ").replace('"', "") + target = target.strip().replace(r"\ ", " ").replace('"', "") # Link target is relative to the link target = os.path.join(os.path.dirname(linkname), target) @@ -73,23 +73,25 @@ def main(): whence_links = list(zip(*links_list))[0] known_files = set(name for name in whence_list if not name.endswith("/")) | set( [ - ".gitignore", ".codespell.cfg", + ".editorconfig", + ".gitignore", ".gitlab-ci.yml", ".pre-commit-config.yaml", + "Dockerfile", + "Makefile", + "README.md", + "WHENCE", "build_packages.py", "check_whence.py", "configure", - "Makefile", - "README.md", - "copy-firmware.sh", - "WHENCE", - "Dockerfile", + "contrib/process_linux_firmware.py", "contrib/templates/debian.changelog", "contrib/templates/debian.control", "contrib/templates/debian.copyright", "contrib/templates/rpm.spec", - "contrib/process_linux_firmware.py", + "copy-firmware.sh", + "dedup-firmware.sh", ] ) known_prefixes = set(name for name in whence_list if name.endswith("/")) @@ -99,7 +101,7 @@ def main(): sys.stderr.write("E: %s listed in WHENCE as File, but is directory\n" % name) ret = 1 - for name in set(fw for fw in whence_files if whence_files.count(fw) > 1): + for name in set(name for name in whence_files if whence_files.count(name) > 1): sys.stderr.write("E: %s listed in WHENCE twice\n" % name) ret = 1 @@ -107,7 +109,7 @@ def main(): sys.stderr.write("E: %s listed in WHENCE twice\n" % name) ret = 1 - for name in set(link for link in whence_files if os.path.islink(link)): + for name in set(file for file in whence_files if os.path.islink(file)): sys.stderr.write("E: %s listed in WHENCE as File, but is a symlink\n" % name) ret = 1 @@ -115,12 +117,20 @@ def main(): sys.stderr.write("E: %s listed in WHENCE as Link, is in tree\n" % name) ret = 1 + invalid_targets = set(link[0] for link in links_list) + for link, target in sorted(links_list): + if target in invalid_targets: + sys.stderr.write( + "E: target %s of link %s is also a link\n" % (target, link) + ) + ret = 1 + for name in sorted(list(known_files - git_files)): sys.stderr.write("E: %s listed in WHENCE does not exist\n" % name) ret = 1 - # A link can point to another link, or to a file... - valid_targets = set(link[0] for link in links_list) | git_files + # A link can point to a file... + valid_targets = set(git_files) # ... or to a directory for target in set(valid_targets): @@ -131,10 +141,10 @@ def main(): break valid_targets.add(dirname) - for name, target in sorted(links_list): + for link, target in sorted(links_list): if target not in valid_targets: sys.stderr.write( - "E: target %s of link %s in WHENCE" " does not exist\n" % (target, name) + "E: target %s of link %s in WHENCE" " does not exist\n" % (target, link) ) ret = 1 diff --git a/copy-firmware.sh b/copy-firmware.sh index 6c557f23..dcb3b92c 100755 --- a/copy-firmware.sh +++ b/copy-firmware.sh @@ -5,11 +5,19 @@ # verbose=: -prune=no # shellcheck disable=SC2209 compress=cat compext= -skip_dedup=0 +destdir= + +err() { + printf "ERROR: %s\n" "$*" + exit 1 +} + +warn() { + printf "WARNING: %s\n" "$*" +} while test $# -gt 0; do case $1 in @@ -19,15 +27,9 @@ while test $# -gt 0; do shift ;; - -P | --prune) - prune=yes - shift - ;; - --xz) if test "$compext" = ".zst"; then - echo "ERROR: cannot mix XZ and ZSTD compression" - exit 1 + err "cannot mix XZ and ZSTD compression" fi compress="xz --compress --quiet --stdout --check=crc32" compext=".xz" @@ -36,8 +38,7 @@ while test $# -gt 0; do --zstd) if test "$compext" = ".xz"; then - echo "ERROR: cannot mix XZ and ZSTD compression" - exit 1 + err "cannot mix XZ and ZSTD compression" fi # shellcheck disable=SC2209 compress="zstd --compress --quiet --stdout" @@ -45,23 +46,9 @@ while test $# -gt 0; do shift ;; - --ignore-duplicates) - skip_dedup=1 - shift - ;; - - -*) - if test "$compress" = "cat"; then - echo "ERROR: unknown command-line option: $1" - exit 1 - fi - compress="$compress $1" - shift - ;; *) - if test "x$destdir" != "x"; then - echo "ERROR: unknown command-line options: $*" - exit 1 + if test -n "$destdir"; then + err "unknown command-line options: $*" fi destdir="$1" @@ -70,21 +57,19 @@ while test $# -gt 0; do esac done -if [ -z "$destdir" ]; then - echo "ERROR: destination directory was not specified" - exit 1 +if test -z "$destdir"; then + err "destination directory was not specified" fi -if ! command -v rdfind >/dev/null; then - if [ "$skip_dedup" != 1 ]; then - echo "ERROR: rdfind is not installed. Pass --ignore-duplicates to skip deduplication" - exit 1 - fi +if test -d "$destdir"; then + find "$destdir" -type d -empty >/dev/null || warn "destination folder is not empty." fi +$verbose "Checking that WHENCE file is formatted properly" +./check_whence.py || err "check_whence.py has detected errors." + # shellcheck disable=SC2162 # file/folder name can include escaped symbols grep -E '^(RawFile|File):' WHENCE | sed -E -e 's/^(RawFile|File): */\1 /;s/"//g' | while read k f; do - test -f "$f" || continue install -d "$destdir/$(dirname "$f")" $verbose "copying/compressing file $f$compext" if test "$compress" != "cat" && test "$k" = "RawFile"; then @@ -95,59 +80,23 @@ grep -E '^(RawFile|File):' WHENCE | sed -E -e 's/^(RawFile|File): */\1 /;s/"//g' fi done -if [ "$skip_dedup" != 1 ] ; then - $verbose "Finding duplicate files" - rdfind -makesymlinks true -makeresultsfile false "$destdir" >/dev/null - find "$destdir" -type l | while read -r l; do - target="$(realpath "$l")" - $verbose "Correcting path for $l" - ln -fs "$(realpath --relative-to="$(dirname "$(realpath -s "$l")")" "$target")" "$l" - done -fi - # shellcheck disable=SC2162 # file/folder name can include escaped symbols -grep -E '^Link:' WHENCE | sed -e 's/^Link: *//g;s/-> //g' | while read f d; do - if test -L "$f$compext"; then - test -f "$destdir/$f$compext" && continue - $verbose "copying link $f$compext" - install -d "$destdir/$(dirname "$f")" - cp -d "$f$compext" "$destdir/$f$compext" - - if test "x$d" != "x"; then - target="$(readlink "$f")" - - if test "x$target" != "x$d"; then - $verbose "WARNING: inconsistent symlink target: $target != $d" - else - if test "x$prune" != "xyes"; then - $verbose "WARNING: unneeded symlink detected: $f" - else - $verbose "WARNING: pruning unneeded symlink $f" - rm -f "$f$compext" - fi - fi - else - $verbose "WARNING: missing target for symlink $f" - fi +grep -E '^Link:' WHENCE | sed -e 's/^Link: *//g;s/-> //g' | while read l t; do + directory="$destdir/$(dirname "$l")" + install -d "$directory" + target="$(cd "$directory" && realpath -m -s "$t")" + if test -e "$target"; then + $verbose "creating link $l -> $t" + ln -s "$t" "$destdir/$l" else - directory="$destdir/$(dirname "$f")" - install -d "$directory" - target="$(cd "$directory" && realpath -m -s "$d")" - if test -e "$target"; then - $verbose "creating link $f -> $d" - ln -s "$d" "$destdir/$f" - else - $verbose "creating link $f$compext -> $d$compext" - ln -s "$d$compext" "$destdir/$f$compext" - fi + $verbose "creating link $l$compext -> $t$compext" + ln -s "$t$compext" "$destdir/$l$compext" fi done # Verify no broken symlinks if test "$(find "$destdir" -xtype l | wc -l)" -ne 0 ; then - echo "ERROR: Broken symlinks found:" - find "$destdir" -xtype l - exit 1 + err "Broken symlinks found:\\n$(find "$destdir" -xtype l)" fi exit 0 diff --git a/dedup-firmware.sh b/dedup-firmware.sh new file mode 100755 index 00000000..2bbd637f --- /dev/null +++ b/dedup-firmware.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Deduplicate files in a given destdir +# + +err() { + echo "ERROR: $*" + exit 1 +} + +verbose=: +destdir= +while test $# -gt 0; do + case $1 in + -v | --verbose) + # shellcheck disable=SC2209 + verbose=echo + ;; + *) + if test -n "$destdir"; then + err "unknown command-line options: $*" + fi + + destdir="$1" + shift + ;; + esac +done + +if test -z "$destdir"; then + err "destination directory was not specified." +fi + +if ! test -d "$destdir"; then + err "provided directory does not exit." +fi + +if ! command -v rdfind >/dev/null; then + err "rdfind is not installed." +fi + +$verbose "Finding duplicate files" +rdfind -makesymlinks true -makeresultsfile true "$destdir" >/dev/null + +grep DUPTYPE_WITHIN_SAME_TREE results.txt | grep -o "$destdir.*" | while read -r l; do + target="$(realpath "$l")" + $verbose "Correcting path for $l" + ln --force --symbolic --relative "$target" "$l" +done + +rm results.txt