From 92186b70ca9c520fc726e0885aac633aadace655 Mon Sep 17 00:00:00 2001 From: Natanael Copa Date: Wed, 26 Aug 2015 08:14:36 +0200 Subject: [PATCH] abuild: fix fetch lock file on nfs flock(2) on an NFS mount will on the server side convert the lock to a POSIX lock (fcntl(F_SETLK)). This means that abuild running on NFS server and client will create different locks and they will both try download same file at same time. We fix this by creating a small abuild-fetch application that will create a POSIX lock which works with NFS. --- Makefile | 6 +- abuild-fetch.c | 194 +++++++++++++++++++++++++++++++++++++++++++++++++ abuild.in | 69 +----------------- 3 files changed, 200 insertions(+), 69 deletions(-) create mode 100644 abuild-fetch.c diff --git a/Makefile b/Makefile index 81c2851..b7332b2 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ abuildrepo ?= ~/.cache/abuild SCRIPTS := abuild abuild-keygen abuild-sign newapkbuild \ abump apkgrel buildlab apkbuild-cpan checkapk \ apkbuild-gem-resolver -USR_BIN_FILES := $(SCRIPTS) abuild-tar abuild-sudo +USR_BIN_FILES := $(SCRIPTS) abuild-tar abuild-sudo abuild-fetch SAMPLES := sample.APKBUILD sample.initd sample.confd \ sample.pre-install sample.post-install AUTOTOOLS_TOOLCHAIN_FILES := config.sub @@ -47,6 +47,7 @@ LIBS-abuild-tar = $(SSL_LIBS) CFLAGS-abuild-tar = $(SSL_CFLAGS) OBJS-abuild-sudo = abuild-sudo.o +OBJS-abuild-fetch = abuild-fetch.o .SUFFIXES: .sh.in .in %.sh: %.sh.in @@ -73,6 +74,9 @@ abuild-sudo: abuild-sudo.o abuild-tar: abuild-tar.o $(LINK) +abuild-fetch: abuild-fetch.o + $(LINK) + abuild-tar.static: abuild-tar.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CFLAGS-$@) -o $@ -static $(LIBS-$@) $^ diff --git a/abuild-fetch.c b/abuild-fetch.c new file mode 100644 index 0000000..51b41eb --- /dev/null +++ b/abuild-fetch.c @@ -0,0 +1,194 @@ +/* MIT license + +Copyright (C) 2015 Natanael Copa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + + +#include + +#include +#include +#include +#include +#include +#include + +static char *program; +static char lockfile[PATH_MAX] = ""; + +struct cmdarray { + size_t argc; + char *argv[32]; +}; + +void add_opt(struct cmdarray *cmd, char *opt) +{ + cmd->argv[cmd->argc++] = opt; + cmd->argv[cmd->argc] = NULL; +} + +int usage(int eval) +{ + printf("usage: %s [-d DESTDIR] URL\n", program); + return eval; +} + +/* create or wait for an NFS-safe lockfile and fetch url with curl or wget */ +int fetch(char *url, const char *destdir) +{ + int lockfd, status=0; + pid_t childpid; + char outfile[PATH_MAX], partfile[PATH_MAX]; + char *name, *p; + struct flock fl = { + .l_type = F_WRLCK, + .l_whence = SEEK_SET, + .l_start = 1, + .l_len = 0, + }; + struct cmdarray curlcmd = { + .argc = 6, + .argv = { "curl", "-k", "-L", "-f", "-o", partfile, NULL } + }; + struct cmdarray wgetcmd = { + .argc = 3, + .argv = { "wget", "-O", partfile, NULL } + }; + + name = strrchr(url, '/'); + if (name == NULL) + errx(1, "%s: no '/' in url"); + p = strstr(url, "::"); + if (p != NULL) { + name = url; + *p = '\0'; + url = p + 2; + } else if (strstr(url, "saveas-") == url) { + *name++ = '\0'; + url += 7; /* strlen("saveas-") */ + } else { + name++; + } + + snprintf(outfile, sizeof(outfile), "%s/%s", destdir, name); + snprintf(lockfile, sizeof(lockfile), "%s.lock", outfile); + snprintf(partfile, sizeof(partfile), "%s.part", outfile); + + lockfd = open(lockfile, O_WRONLY|O_CREAT, 0660); + if (lockfd < 0) + err(1, lockfile); + + if (fcntl(lockfd, F_SETLK, &fl) < 0) { + printf("Waiting for %s ...\n", lockfile); + if (fcntl(lockfd, F_SETLKW, &fl) < 0) + err(1, "fcntl(F_SETLKW)"); + } + + if (access(outfile, F_OK) == 0) + goto fetch_done; + + if (access(partfile, F_OK) == 0) { + printf("Partial download found. Trying to resume.\n"); + add_opt(&curlcmd, "-C"); + add_opt(&curlcmd, "-"); + add_opt(&wgetcmd, "-c"); + } + + add_opt(&curlcmd, url); + add_opt(&wgetcmd, url); + + childpid = fork(); + if (childpid < 0 ) + err(1, "fork"); + + if (childpid == 0) { + execvp(curlcmd.argv[0], curlcmd.argv); + printf("Using wget\n"); + execvp(wgetcmd.argv[0], wgetcmd.argv); + warn(wgetcmd.argv[0]); + unlink(lockfile); + exit(1); + } + wait(&status); + rename(partfile, outfile); + +fetch_done: + unlink(lockfile); + close(lockfd); + lockfile[0] = '\0'; + return status; + +} + +void sighandler(int sig) +{ + switch(sig) { + case SIGABRT: + case SIGINT: + case SIGQUIT: + case SIGTERM: + unlink(lockfile); + exit(0); + break; + default: + break; + } +} + +int main(int argc, char *argv[]) +{ + int opt, r=0, i; + char *destdir = "/var/cache/distfiles"; + + program = argv[0]; + while ((opt = getopt(argc, argv, "hd:")) != -1) { + switch (opt) { + case 'h': + return usage(0); + break; + case 'd': + destdir = optarg; + break; + default: + printf("Unkonwn option '%c'\n", opt); + return usage(1); + break; + } + } + + argv += optind; + argc -= optind; + + if (argc < 1) + return usage(1); + + signal(SIGABRT, sighandler); + signal(SIGINT, sighandler); + signal(SIGQUIT, sighandler); + signal(SIGTERM, sighandler); + + for (i = 0; i < argc; i++) { + if (fetch(argv[i], destdir)) + r++; + } + return r; +} diff --git a/abuild.in b/abuild.in index b8d8851..952048b 100644 --- a/abuild.in +++ b/abuild.in @@ -298,79 +298,12 @@ sourcecheck() { return 0 } -# convert curl options to wget options and call wget instead of curl -wget_fallback() { - local wget_opts= outfile= opt= - while getopts "C:Lko:s" opt; do - case $opt in - 'L') ;; # --location. wget does this by default - 'f') ;; # --fail. wget does this by default - 'C') wget_opts="$wget_opts -c";; # --continue-at - 's') wget_opts="$wget_opts -q";; # --silent - 'o') wget_opts="$wget_opts -O $OPTARG";; # --output - 'k') wget_opts="$wget_opts --no-check-certificate";; #gnu wget - esac - done - shift $(( $OPTIND - 1 )) - wget $wget_opts "$1" -} - uri_fetch() { local uri="$1" - local d="${uri##*/}" # $(basename $uri) - local opts [ -n "$quiet" ] && opts="-s" - local lockfile="$SRCDEST/$d".lock - - # fix saveas-*://* URIs - case "$uri" in - # remove 'saveas-' from beginning and - # '/filename' from end of URI - saveas-*://*) uri="${uri:7:$(expr ${#uri} - 7 - ${#d} - 1)}";; - - *::*) - d=${uri%%::*} - uri=${uri#$d::} - ;; - esac - - case "$uri" in - https://*) opts="-k";; - esac - mkdir -p "$SRCDEST" - - CLEANUP_FILES="$CLEANUP_FILES $lockfile" - ( - flock -n -x 9 || msg "Waiting for ${lockfile##*/}..." - flock -x 9 - - [ -f "$SRCDEST/$d" ] && exit 0 # use exit since its a subshell - - if [ -f "$SRCDEST/$d.part" ]; then - msg "Partial download found. Trying to resume" - opts="$opts -C -" - fi - msg "Fetching $uri" - - # fallback to wget if curl is missing. useful for bootstrapping - local fetcher= - if ! [ -x "$(which curl)" ]; then - fetcher=wget_fallback - else - fetcher=curl - opts="$opts -L -f -k" - fi - - $fetcher $opts -o "$SRCDEST/$d.part" "$uri" \ - && mv "$SRCDEST/$d.part" "$SRCDEST/$d" - - ) 9>$lockfile - - local rc=$? - rm -f "$lockfile" - return $rc + abuild-fetch -d "$SRCDEST" "$uri" } is_remote() {