abuild: fix fetch lock file on nfs
flock(2) on an NFS mount will on the server side convert the lock to a POSIX lock (fcntl(F_SETLK)). This means that abuild running on NFS server and client will create different locks and they will both try download same file at same time. We fix this by creating a small abuild-fetch application that will create a POSIX lock which works with NFS.
This commit is contained in:
Natanael Copa
parent
bed1c80a46
commit
92186b70ca
6
Makefile
6
Makefile
|
|
@ -11,7 +11,7 @@ abuildrepo ?= ~/.cache/abuild
|
|||
SCRIPTS := abuild abuild-keygen abuild-sign newapkbuild \
|
||||
abump apkgrel buildlab apkbuild-cpan checkapk \
|
||||
apkbuild-gem-resolver
|
||||
USR_BIN_FILES := $(SCRIPTS) abuild-tar abuild-sudo
|
||||
USR_BIN_FILES := $(SCRIPTS) abuild-tar abuild-sudo abuild-fetch
|
||||
SAMPLES := sample.APKBUILD sample.initd sample.confd \
|
||||
sample.pre-install sample.post-install
|
||||
AUTOTOOLS_TOOLCHAIN_FILES := config.sub
|
||||
|
|
@ -47,6 +47,7 @@ LIBS-abuild-tar = $(SSL_LIBS)
|
|||
CFLAGS-abuild-tar = $(SSL_CFLAGS)
|
||||
|
||||
OBJS-abuild-sudo = abuild-sudo.o
|
||||
OBJS-abuild-fetch = abuild-fetch.o
|
||||
|
||||
.SUFFIXES: .sh.in .in
|
||||
%.sh: %.sh.in
|
||||
|
|
@ -73,6 +74,9 @@ abuild-sudo: abuild-sudo.o
|
|||
abuild-tar: abuild-tar.o
|
||||
$(LINK)
|
||||
|
||||
abuild-fetch: abuild-fetch.o
|
||||
$(LINK)
|
||||
|
||||
abuild-tar.static: abuild-tar.o
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(CFLAGS-$@) -o $@ -static $(LIBS-$@) $^
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
/* MIT license
|
||||
|
||||
Copyright (C) 2015 Natanael Copa <ncopa@alpinelinux.org>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
*/
|
||||
|
||||
|
||||
#include <sys/wait.h>
|
||||
|
||||
#include <err.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static char *program;
|
||||
static char lockfile[PATH_MAX] = "";
|
||||
|
||||
struct cmdarray {
|
||||
size_t argc;
|
||||
char *argv[32];
|
||||
};
|
||||
|
||||
void add_opt(struct cmdarray *cmd, char *opt)
|
||||
{
|
||||
cmd->argv[cmd->argc++] = opt;
|
||||
cmd->argv[cmd->argc] = NULL;
|
||||
}
|
||||
|
||||
int usage(int eval)
|
||||
{
|
||||
printf("usage: %s [-d DESTDIR] URL\n", program);
|
||||
return eval;
|
||||
}
|
||||
|
||||
/* create or wait for an NFS-safe lockfile and fetch url with curl or wget */
|
||||
int fetch(char *url, const char *destdir)
|
||||
{
|
||||
int lockfd, status=0;
|
||||
pid_t childpid;
|
||||
char outfile[PATH_MAX], partfile[PATH_MAX];
|
||||
char *name, *p;
|
||||
struct flock fl = {
|
||||
.l_type = F_WRLCK,
|
||||
.l_whence = SEEK_SET,
|
||||
.l_start = 1,
|
||||
.l_len = 0,
|
||||
};
|
||||
struct cmdarray curlcmd = {
|
||||
.argc = 6,
|
||||
.argv = { "curl", "-k", "-L", "-f", "-o", partfile, NULL }
|
||||
};
|
||||
struct cmdarray wgetcmd = {
|
||||
.argc = 3,
|
||||
.argv = { "wget", "-O", partfile, NULL }
|
||||
};
|
||||
|
||||
name = strrchr(url, '/');
|
||||
if (name == NULL)
|
||||
errx(1, "%s: no '/' in url");
|
||||
p = strstr(url, "::");
|
||||
if (p != NULL) {
|
||||
name = url;
|
||||
*p = '\0';
|
||||
url = p + 2;
|
||||
} else if (strstr(url, "saveas-") == url) {
|
||||
*name++ = '\0';
|
||||
url += 7; /* strlen("saveas-") */
|
||||
} else {
|
||||
name++;
|
||||
}
|
||||
|
||||
snprintf(outfile, sizeof(outfile), "%s/%s", destdir, name);
|
||||
snprintf(lockfile, sizeof(lockfile), "%s.lock", outfile);
|
||||
snprintf(partfile, sizeof(partfile), "%s.part", outfile);
|
||||
|
||||
lockfd = open(lockfile, O_WRONLY|O_CREAT, 0660);
|
||||
if (lockfd < 0)
|
||||
err(1, lockfile);
|
||||
|
||||
if (fcntl(lockfd, F_SETLK, &fl) < 0) {
|
||||
printf("Waiting for %s ...\n", lockfile);
|
||||
if (fcntl(lockfd, F_SETLKW, &fl) < 0)
|
||||
err(1, "fcntl(F_SETLKW)");
|
||||
}
|
||||
|
||||
if (access(outfile, F_OK) == 0)
|
||||
goto fetch_done;
|
||||
|
||||
if (access(partfile, F_OK) == 0) {
|
||||
printf("Partial download found. Trying to resume.\n");
|
||||
add_opt(&curlcmd, "-C");
|
||||
add_opt(&curlcmd, "-");
|
||||
add_opt(&wgetcmd, "-c");
|
||||
}
|
||||
|
||||
add_opt(&curlcmd, url);
|
||||
add_opt(&wgetcmd, url);
|
||||
|
||||
childpid = fork();
|
||||
if (childpid < 0 )
|
||||
err(1, "fork");
|
||||
|
||||
if (childpid == 0) {
|
||||
execvp(curlcmd.argv[0], curlcmd.argv);
|
||||
printf("Using wget\n");
|
||||
execvp(wgetcmd.argv[0], wgetcmd.argv);
|
||||
warn(wgetcmd.argv[0]);
|
||||
unlink(lockfile);
|
||||
exit(1);
|
||||
}
|
||||
wait(&status);
|
||||
rename(partfile, outfile);
|
||||
|
||||
fetch_done:
|
||||
unlink(lockfile);
|
||||
close(lockfd);
|
||||
lockfile[0] = '\0';
|
||||
return status;
|
||||
|
||||
}
|
||||
|
||||
void sighandler(int sig)
|
||||
{
|
||||
switch(sig) {
|
||||
case SIGABRT:
|
||||
case SIGINT:
|
||||
case SIGQUIT:
|
||||
case SIGTERM:
|
||||
unlink(lockfile);
|
||||
exit(0);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
int opt, r=0, i;
|
||||
char *destdir = "/var/cache/distfiles";
|
||||
|
||||
program = argv[0];
|
||||
while ((opt = getopt(argc, argv, "hd:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'h':
|
||||
return usage(0);
|
||||
break;
|
||||
case 'd':
|
||||
destdir = optarg;
|
||||
break;
|
||||
default:
|
||||
printf("Unkonwn option '%c'\n", opt);
|
||||
return usage(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
argv += optind;
|
||||
argc -= optind;
|
||||
|
||||
if (argc < 1)
|
||||
return usage(1);
|
||||
|
||||
signal(SIGABRT, sighandler);
|
||||
signal(SIGINT, sighandler);
|
||||
signal(SIGQUIT, sighandler);
|
||||
signal(SIGTERM, sighandler);
|
||||
|
||||
for (i = 0; i < argc; i++) {
|
||||
if (fetch(argv[i], destdir))
|
||||
r++;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
69
abuild.in
69
abuild.in
|
|
@ -298,79 +298,12 @@ sourcecheck() {
|
|||
return 0
|
||||
}
|
||||
|
||||
# convert curl options to wget options and call wget instead of curl
|
||||
wget_fallback() {
|
||||
local wget_opts= outfile= opt=
|
||||
while getopts "C:Lko:s" opt; do
|
||||
case $opt in
|
||||
'L') ;; # --location. wget does this by default
|
||||
'f') ;; # --fail. wget does this by default
|
||||
'C') wget_opts="$wget_opts -c";; # --continue-at
|
||||
's') wget_opts="$wget_opts -q";; # --silent
|
||||
'o') wget_opts="$wget_opts -O $OPTARG";; # --output
|
||||
'k') wget_opts="$wget_opts --no-check-certificate";; #gnu wget
|
||||
esac
|
||||
done
|
||||
shift $(( $OPTIND - 1 ))
|
||||
wget $wget_opts "$1"
|
||||
}
|
||||
|
||||
uri_fetch() {
|
||||
local uri="$1"
|
||||
local d="${uri##*/}" # $(basename $uri)
|
||||
local opts
|
||||
[ -n "$quiet" ] && opts="-s"
|
||||
|
||||
local lockfile="$SRCDEST/$d".lock
|
||||
|
||||
# fix saveas-*://* URIs
|
||||
case "$uri" in
|
||||
# remove 'saveas-' from beginning and
|
||||
# '/filename' from end of URI
|
||||
saveas-*://*) uri="${uri:7:$(expr ${#uri} - 7 - ${#d} - 1)}";;
|
||||
|
||||
*::*)
|
||||
d=${uri%%::*}
|
||||
uri=${uri#$d::}
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$uri" in
|
||||
https://*) opts="-k";;
|
||||
esac
|
||||
|
||||
mkdir -p "$SRCDEST"
|
||||
|
||||
CLEANUP_FILES="$CLEANUP_FILES $lockfile"
|
||||
(
|
||||
flock -n -x 9 || msg "Waiting for ${lockfile##*/}..."
|
||||
flock -x 9
|
||||
|
||||
[ -f "$SRCDEST/$d" ] && exit 0 # use exit since its a subshell
|
||||
|
||||
if [ -f "$SRCDEST/$d.part" ]; then
|
||||
msg "Partial download found. Trying to resume"
|
||||
opts="$opts -C -"
|
||||
fi
|
||||
msg "Fetching $uri"
|
||||
|
||||
# fallback to wget if curl is missing. useful for bootstrapping
|
||||
local fetcher=
|
||||
if ! [ -x "$(which curl)" ]; then
|
||||
fetcher=wget_fallback
|
||||
else
|
||||
fetcher=curl
|
||||
opts="$opts -L -f -k"
|
||||
fi
|
||||
|
||||
$fetcher $opts -o "$SRCDEST/$d.part" "$uri" \
|
||||
&& mv "$SRCDEST/$d.part" "$SRCDEST/$d"
|
||||
|
||||
) 9>$lockfile
|
||||
|
||||
local rc=$?
|
||||
rm -f "$lockfile"
|
||||
return $rc
|
||||
abuild-fetch -d "$SRCDEST" "$uri"
|
||||
}
|
||||
|
||||
is_remote() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue