Merge pull request #9853 from dx9/wip-alpine-dev-support

alpine: add alpine linux dev support
This commit is contained in:
Sage Weil 2017-04-21 18:54:25 -05:00 committed by GitHub
commit 15e80fa2d8
12 changed files with 608 additions and 9 deletions

4
.gitignore vendored
View File

@ -31,6 +31,10 @@ core
*.log
/src/td
# alpine abuild things
.abuild/
alpine/APKBUILD
# specific local dir files
/build-doc
/tags

38
README.alpine.md Normal file
View File

@ -0,0 +1,38 @@
# Alpine Build (Experimental)
## Dev Env Setup
```
apk --update add bash sudo git
git clone https://github.com/ceph/ceph
```
### Build
```
./run-make-check.sh -DWITH_EMBEDDED=OFF -DWITH_SYSTEM_BOOST=ON -DWITH_LTTNG=OFF -DWITH_REENTRANT_STRSIGNAL=ON -DWITH_THREAD_SAFE_RES_QUERY=ON
```
### Packaging
```
./make-apk.sh
```
### Docker
```
cd ceph/src
./test/docker-test.sh --os-type alpine --os-version edge ./make-apk.sh
or
./test/docker-test.sh --os-type alpine --os-version edge -- ./run-make-check.sh -DWITH_EMBEDDED=OFF -DWITH_SYSTEM_BOOST=ON -DWITH_LTTNG=OFF -DWITH_REENTRANT_STRSIGNAL=ON -DWITH_THREAD_SAFE_RES_QUERY=ON
```
## Known Issues
- Uses musl libc malloc because musl does not currently support replacing malloc implementation. see https://bugs.alpinelinux.org/issues/5389
- No backtrace support need to look at libunwind integration

14
alpine/.editorconfig Normal file
View File

@ -0,0 +1,14 @@
; This file is for unifying the coding style for different editors and IDEs.
; Plugins are available for notepad++, emacs, vim, gedit,
; textmate, visual studio, and more.
;
; See http://editorconfig.org for details.
# Top-most EditorConfig file.
root = true
[**/APKBUILD*]
indent_style = tab
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true

485
alpine/APKBUILD.in Normal file
View File

@ -0,0 +1,485 @@
# Contributor: John Coyle <dx9err@gmail.com>
# Maintainer: John Coyle <dx9err@gmail.com>
pkgname=ceph
pkgver=@VERSION@
pkgrel=@RPM_RELEASE@
pkgdesc="Ceph is a distributed object store and file system"
pkgusers="ceph"
pkggroups="ceph"
url="http://ceph.com"
arch="x86_64"
license="LGPL-2.1 and CC-BY-SA-1.0 and GPL-2.0 and BSL-1.0 and \
GPL-2.0-with-autoconf-exception and BSD-3-Clause and MIT"
depends="ceph-osd ceph-mds ceph-mgr ceph-mon"
# grep --quiet option required
# diffutils -y option required by osd-scrub-snaps.sh
# util-linux uuidgen required
makedepends="
acl-dev
argp-standalone
bc
boost-dev
btrfs-progs
bzip2-dev
cmake
coreutils
cryptsetup
curl-dev
cython-dev
diffutils
eudev-dev
expat-dev
fcgi-dev
flex
fuse-dev
git
grep
jq
keyutils-dev
leveldb-dev
libaio-dev
libatomic_ops-dev
libedit-dev
libressl-dev
libtirpc-dev
libtool
libxml2-dev
linux-headers
lvm2-dev
nss-dev
openldap-dev
parted
procps
python-dev
py-pip
py-nose
py-sphinx
py-virtualenv
readline-dev
rpcgen
snappy-dev
userspace-rcu-dev
util-linux
xfsprogs-dev
xmlstarlet
yasm
"
source="@TARBALL_BASENAME@.tar.bz2"
subpackages="
$pkgname-base
$pkgname-common
$pkgname-mds
$pkgname-mgr
$pkgname-mon
$pkgname-fuse:ceph_fuse
$pkgname-radosgw
$pkgname-osd
$pkgname-doc
$pkgname-bash-completion:bash_completion
rbd-fuse:rbd_fuse
rbd-mirror:rbd_mirror
rbd-nbd:rbd_nbd
librbd
librbd-dev:librbd_dev
libcephfs
libcephfs-dev:libcephfs_dev
librados
librados-dev:librados_dev
librgw
librgw-dev:librgw_dev
py-rados:py_rados
py-rgw:py_rgw
libradosstriper
libradosstriper-dev:libradosstriper_dev
py-rbd:py_rbd
py-cephfs:py_cephfs
"
if [ -n "$CEPH_TEST_PKG" ]; then
subpackages="$subpackages $pkgname-test:ceph_test"
fi
_ceph_uid=167
_ceph_gid=167
_prefix=/usr
_bindir=$_prefix/bin
_datadir=$_prefix/share
_docdir=$_datadir/doc
_includedir=$_prefix/include
_libdir=$_prefix/lib
_libexecdir=$_prefix/libexec
_localstatedir=/var
_mandir=$_datadir/man
_sbindir=/usr/sbin
_sysconfdir=/etc
_udevrulesdir=/etc/udev/rules.d
_python_sitelib=/usr/lib/python2.7/site-packages
builddir=$srcdir/@TARBALL_BASENAME@
build() {
export CEPH_BUILD_VIRTUALENV=$builddir
mkdir -p $builddir/build
cd $builddir/build
if [ -n "$MAKE_CHECK" ] || [ -n "$CEPH_TEST_PKG" ]; then
local _with_tests=ON
fi
cmake .. \
-DCMAKE_INSTALL_PREFIX=$_prefix \
-DCMAKE_INSTALL_LIBDIR=$_libdir \
-DCMAKE_INSTALL_LIBEXECDIR=$_libexecdir \
-DCMAKE_INSTALL_LOCALSTATEDIR=$_localstatedir \
-DCMAKE_INSTALL_SYSCONFDIR=$_sysconfdir \
-DCMAKE_INSTALL_DOCDIR=$_docdir/ceph \
-DCMAKE_INSTALL_MANDIR=$_mandir \
-DWITH_REENTRANT_STRSIGNAL=ON \
-DWITH_THREAD_SAFE_RES_QUERY=ON \
-DWITH_MANPAGE=ON \
-DWITH_PYTHON3=OFF \
-DWITH_LTTNG=OFF \
-DWITH_SYSTEM_BOOST=ON \
-DWITH_EMBEDDED=OFF \
-DWITH_TESTS=${_with_tests:-OFF} \
|| return 1
make -j${JOBS:-2} || return 1
if [ -n "$MAKE_CHECK" ]; then
ctest -j${JOBS:-2}
fi
}
package() {
cd $builddir/build
make DESTDIR=$pkgdir install || return 1
cd ..
rm -f $pkgdir$_sysconfdir/init.d/ceph
install -m 0644 -D src/etc-rbdmap $pkgdir$_sysconfdir/ceph/rbdmap \
|| return 1
install -m 0644 -D src/logrotate.conf $pkgdir$_sysconfdir/logrotate.d/ceph \
|| return 1
chmod 0644 $pkgdir$_docdir/ceph/sample.ceph.conf || return 1
chmod 0644 $pkgdir$_docdir/ceph/sample.fetch_config || return 1
# udev rules
install -m 0644 -D udev/50-rbd.rules $pkgdir$_udevrulesdir/50-rbd.rules || return 1
install -m 0644 -D udev/60-ceph-by-parttypeuuid.rules \
$pkgdir$_udevrulesdir/60-ceph-by-parttypeuuid.rules || return 1
install -m 0644 -D udev/95-ceph-osd.rules \
$pkgdir$_udevrulesdir/95-ceph-osd.rules
}
base() {
pkgdesc="Base is the package that includes all the files shared amongst ceph servers"
depends="
ceph-common
cryptsetup
librbd
librados
libcephfs
librgw
logrotate
py-requests
py-setuptools
util-linux
xfsprogs
"
_pkg $_docdir/ceph sample.ceph.conf sample.fetch_config
_pkg $_bindir crushtool monmaptool osdmaptool ceph-run ceph-detect-init
_pkg $_sbindir ceph-create-keys mount.ceph
_pkg $_libexecdir/ceph ceph_common.sh
_pkg $_libdir/rados-classes *.so*
_pkg $_libdir/ceph/erasure-code libec_*.so*
_pkg $_libdir/ceph/compressor libceph_*.so*
_pkg $_sysconfdir/logrotate.d ceph
_pkg $_python_sitelib ceph_detect_init* ceph_disk*
for dir in tmp bootstrap-osd bootstrap-mds bootstrap-rgw; do
install -m 750 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/lib/ceph/$dir || return 1
done
}
common() {
pkgdesc="Common utilities to mount and interact with a ceph storage cluster."
depends="py-rados py-rbd py-cephfs"
install="$pkgname-common.pre-install"
_pkg $_bindir ceph \
ceph-authtool \
ceph-conf \
ceph-dencoder \
ceph-rbdnamer \
ceph-syn \
ceph-crush-location \
cephfs-data-scan \
cephfs-journal-tool \
cephfs-table-tool \
rados \
rbd \
rbd-replay \
rbd-replay-many \
rbdmap \
ceph-post-file \
ceph-brag
_pkg $_datadir/ceph known_hosts_drop.ceph.com id_rsa_drop.ceph.com \
id_rsa_drop.ceph.com.pub
_pkg $_sysconfdir/ceph rbdmap
_pkg $_python_sitelib ceph_argparse.py* ceph_daemon.py*
_pkg $_udevrulesdir 50-rbd.rules
install -m 3770 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/log/ceph || return 1
install -m 750 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/lib/ceph
}
mds() {
pkgdesc="Metadata server daemon for the Ceph distributed file system."
depends="ceph-base"
_pkg $_bindir ceph-mds
install -m 750 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/lib/ceph/mds
}
mon() {
pkgdesc="Cluster monitor daemon for the Ceph distributed file system."
depends="ceph-base"
_pkg $_bindir ceph-mon ceph-rest-api
_pkg $_python_sitelib ceph_rest_api.py*
install -m 750 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/lib/ceph/mon
}
ceph_fuse() {
pkgdesc="FUSE based client for Ceph distributed network file system."
_pkg $_bindir ceph-fuse
_pkg $_sbindir mount.fuse.ceph
}
rbd_fuse() {
pkgdesc="FUSE based client to map Ceph rbd images to files."
depends="librados librbd"
_pkg $_bindir rbd-fuse
}
rbd_mirror() {
pkgdesc="Daemon for mirroring RBD images between Ceph clusters."
depends="ceph-common librados"
_pkg $_bindir rbd-mirror
}
rbd_nbd() {
pkgdesc="NBD based client to map Ceph rbd images to local device."
depends="librbd librados"
_pkg $_bindir rbd-nbd
}
radosgw() {
pkgdesc="Rados REST gateway which implements Amazon's S3 and OpenStack's Swift APIs."
depends="ceph-common"
_pkg $_bindir radosgw radosgw-admin radosgw-token radosgw-object-expirer
mkdir -p $subpkgdir$_localstatedir/lib/ceph/radosgw
}
osd() {
pkgdesc="Object storage daemon for the Ceph distributed file system."
depends="ceph-base parted gptfdisk"
_pkg $_bindir ceph-clsinfo ceph-bluefs-tool ceph-objectstore-tool ceph-osd
_pkg $_sbindir ceph-disk ceph-disk-udev
_pkg $_libexecdir/ceph ceph-osd-prestart.sh
_pkg $_udevrulesdir 60-ceph-by-parttypeuuid.rules 95-ceph-osd.rules
install -m 750 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/lib/ceph/osd
}
librados() {
pkgdesc="RADOS distributed object store client library"
_pkg $_libdir librados.so.*
}
librados_dev() {
pkgdesc="RADOS distributed object store client library headers"
depends="librados"
_pkg $_includedir/rados librados.h \
librados.hpp \
buffer.h \
buffer_fwd.h \
inline_memory.h \
page.h \
crc32c.h \
rados_types.h \
rados_types.hpp \
memory.h
_pkg $_libdir librados.so
_pkg $_bindir librados-config
}
librgw() {
pkgdesc="RADOS gateway client library"
depends="librados"
_pkg $_libdir librgw.so.*
}
librgw_dev() {
pkgdesc="RADOS gateway client library headers"
depends="librados"
_pkg $_includedir/rados librgw.h rgw_file.h
_pkg $_libdir librgw.so
}
py_rgw() {
pkgdesc="Python 2 libraries for the RADOS gateway"
depends="librgw py-rados"
_pkg $_python_sitelib rgw.so rgw-*.egg-info
}
py_rados() {
pkgdesc="Python libraries for the RADOS object store"
depends="librados"
_pkg $_python_sitelib rados.so rados-*.egg-info
}
libradosstriper() {
pkgdesc="RADOS striping library"
depends="librados"
_pkg $_libdir libradosstriper.so.*
}
libradosstriper_dev() {
pkgdesc="RADOS striping library headers"
depends="libradosstriper librados-dev"
_pkg $_includedir/radosstriper libradosstriper.h libradosstriper.hpp
_pkg $_libdir libradosstriper.so
}
librbd() {
pkgdesc="RADOS block device client library"
depends="librados"
_pkg $_libdir librbd.so.*
}
librbd_dev() {
pkgdesc="RADOS block device client library headers"
depends="librbd librados-dev"
_pkg $_includedir/rbd features.h librbd.h librbd.hpp
_pkg $_libdir librbd.so
}
py_rbd() {
pkgdesc="Python libraries for the RADOS block device"
depends="librbd py-rados"
_pkg $_python_sitelib rbd.so rbd-*.egg-info
}
libcephfs() {
pkgdesc="Ceph distributed file system client library"
_pkg $_libdir libcephfs.so.*
}
libcephfs_dev() {
pkgdesc="Ceph distributed file system client library headers"
depends="libcephfs librados-devel"
_pkg $_includedir/cephfs ceph_statx.h libcephfs.h
_pkg $_libdir libcephfs.so
}
py_cephfs() {
pkgdesc="Python libraries for Ceph distributed file system"
depends="libcephfs py-rados"
_pkg $_python_sitelib cephfs.so cephfs-*.egg-info ceph_volume_client.py*
}
ceph_test() {
pkgdesc="Ceph benchmarks and test tools"
depends="ceph-common xmlstarlet"
_pkg $_bindir ceph-client-debug \
ceph_bench_log \
ceph_kvstorebench \
ceph_multi_stress_watch \
ceph_erasure_code \
ceph_erasure_code_benchmark \
ceph_omapbench \
ceph_objectstore_bench \
ceph_perf_objectstore \
ceph_perf_local \
ceph_perf_msgr_client \
ceph_perf_msgr_server \
ceph_psim \
ceph_radosacl \
ceph_rgw_jsonparser \
ceph_rgw_multiparser \
ceph_scratchtool \
ceph_scratchtoolpp \
ceph_smalliobench \
ceph_smalliobenchdumb \
ceph_smalliobenchfs \
ceph_smalliobenchrbd \
ceph_test_* \
ceph_tpbench \
ceph_xattr_bench \
ceph-coverage \
ceph-monstore-tool \
ceph-osdomap-tool \
ceph-kvstore-tool \
ceph-debugpack
_pkg $_libdir ceph/ceph-monstore-update-crush.sh
}
bash_completion() {
depends="bash-completion"
pkgdesc="Bash completions for Ceph"
_pkg $_sysconfdir/bash_completion.d ceph rados radosgw-admin rbd
}
mgr() {
pkgdesc="Ceph Manager Daemon"
depends="ceph-base"
_pkg $_bindir ceph-mgr
_pkg $_libdir/ceph mgr
install -m 750 -o $_ceph_uid -g $_ceph_gid -d \
$subpkgdir$_localstatedir/lib/ceph/mgr
}
_pkg() {
local path=$1
shift
local files=$@
mkdir -p $subpkgdir$path || exit 1
for _file in $files; do
mv $pkgdir$path/$_file $subpkgdir$path || exit 1
done
}

View File

@ -0,0 +1,5 @@
#!/bin/sh
addgroup ceph -g 167 -S 2>/dev/null
adduser ceph -u 167 -S -G ceph -s /sbin/nologin -h /var/lib/ceph -g "Ceph Daemons" 2> /dev/null
exit 0

View File

@ -134,6 +134,18 @@ else
sed -e 's/@//g' < ceph.spec.in > $DIR/ceph.spec
$SUDO zypper --non-interactive install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
;;
alpine)
# for now we need the testing repo for leveldb
TESTREPO="http://nl.alpinelinux.org/alpine/edge/testing"
if ! grep -qF "$TESTREPO" /etc/apk/repositories ; then
$SUDO echo "$TESTREPO" | sudo tee -a /etc/apk/repositories > /dev/null
fi
source alpine/APKBUILD.in
$SUDO apk --update add abuild build-base ccache $makedepends
if id -u build >/dev/null 2>&1 ; then
$SUDO addgroup build abuild
fi
;;
*)
echo "$ID is unknown, dependencies will have to be installed manually."
exit 1

26
make-apk.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/sh
#
# Make Alpine Packages
#
set -xe
#
# make a distribution
#
./make-dist
mv -f *.tar.bz2 ./alpine
#
# alpine packaging key stuff
#
rm -rf .abuild && mkdir -p .abuild
ABUILD_USERDIR=$(pwd)/.abuild abuild-keygen -n -a
source .abuild/abuild.conf
#
# package it
#
cd alpine
abuild checksum && JOBS=$(expr $(nproc) / 2) SRCDEST=$(pwd) REPODEST=$(pwd) PACKAGER_PRIVKEY=$PACKAGER_PRIVKEY abuild -r
cd ..

View File

@ -49,12 +49,15 @@ else
rpm_release=0
fi
cat ceph.spec.in | \
sed "s/@VERSION@/$rpm_version/g" | \
sed "s/@RPM_RELEASE@/$rpm_release/g" |
sed "s/@TARBALL_BASENAME@/ceph-$version/g" > ceph.spec
for spec in ceph.spec.in alpine/APKBUILD.in; do
cat $spec |
sed "s/@VERSION@/$rpm_version/g" |
sed "s/@RPM_RELEASE@/$rpm_release/g" |
sed "s/@TARBALL_BASENAME@/ceph-$version/g" > `echo $spec | sed 's/.in$//'`
done
ln -s . $outfile
tar cvf $outfile.version.tar $outfile/src/.git_version $outfile/ceph.spec
tar cvf $outfile.version.tar $outfile/src/.git_version $outfile/ceph.spec $outfile/alpine/APKBUILD
tar --concatenate -f $outfile.both.tar $outfile.version.tar
tar --concatenate -f $outfile.both.tar $outfile.tar
mv $outfile.both.tar $outfile.tar

View File

@ -27,10 +27,6 @@ function get_processors() {
fi
}
DEFAULT_MAKEOPTS=${DEFAULT_MAKEOPTS:--j$(get_processors)}
BUILD_MAKEOPTS=${BUILD_MAKEOPTS:-$DEFAULT_MAKEOPTS}
CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
function run() {
local install_cmd
if test -f /etc/redhat-release ; then
@ -63,6 +59,12 @@ function run() {
if test -f ./install-deps.sh ; then
$DRY_RUN ./install-deps.sh || return 1
fi
# Init defaults after deps are installed. get_processors() depends on coreutils nproc.
DEFAULT_MAKEOPTS=${DEFAULT_MAKEOPTS:--j$(get_processors)}
BUILD_MAKEOPTS=${BUILD_MAKEOPTS:-$DEFAULT_MAKEOPTS}
CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
$DRY_RUN ./do_cmake.sh $@ || return 1
$DRY_RUN cd build
$DRY_RUN make $BUILD_MAKEOPTS tests || return 1

View File

@ -0,0 +1 @@
../../../alpine/APKBUILD.in

View File

@ -0,0 +1,8 @@
FROM alpine:%%os_version%%
COPY install-deps.sh /root/
COPY APKBUILD.in /root/alpine/APKBUILD.in
RUN apk --update --no-cache add bash sudo
RUN if test %%USER%% != root ; then adduser -D -H -u %%user_id%% %%USER%% && echo '%%USER%% ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers ; fi
# build dependencies
RUN cd /root ; ./install-deps.sh

View File

@ -0,0 +1 @@
../../../install-deps.sh