mirror of
https://github.com/ceph/ceph
synced 2025-02-17 07:57:44 +00:00
5b2de2b9d6
Thanks to huang jun <hjwsm1989@gmail.com> for finding the bug. Signed-off-by: Tommi Virtanen <tommi.virtanen@dreamhost.com>
484 lines
11 KiB
Bash
484 lines
11 KiB
Bash
#!/bin/sh
|
|
#
|
|
# mkcephfs
|
|
#
|
|
# This tool is designed to be flexible. There are two ways to go:
|
|
#
|
|
# The easy way does everything for you using ssh keys. This does not
|
|
# scale well for large clusters.
|
|
#
|
|
# master$ mkcephfs -a -c /etc/ceph/ceph.conf
|
|
#
|
|
# Alternatively, you can use whatever file distribution and/or job
|
|
# launching you want.
|
|
#
|
|
# master$ mkdir /tmp/foo
|
|
# master$ mkcephfs -d /tmp/foo -c /etc/ceph/ceph.conf --prepare-monmap
|
|
#
|
|
# ...copy/share /tmp/foo with all osd and mds nodes at /tmp/bar...
|
|
#
|
|
# osd$ mkcephfs -d /tmp/bar --init-local-daemons osd
|
|
# mds$ mkcephfs -d /tmp/bar --init-local-daemons mds
|
|
#
|
|
# ...gather contents of /tmp/bar's back into /tmp/foo...
|
|
#
|
|
# master$ mkcephfs -d /tmp/foo --prepare-mon
|
|
#
|
|
# ...distribute /tmp/foo to each monitor node...
|
|
#
|
|
# mon$ mkcephfs -d /tmp/foo --init-local-daemons mon
|
|
#
|
|
# master$ cp /tmp/foo/keyring.admin /etc/ceph/keyring # don't forget!
|
|
#
|
|
# In the degenerate case (one node), this is just
|
|
#
|
|
# mkdir /tmp/foo
|
|
# mkcephfs -c ceph.conf -d /tmp/foo --prepare-monmap
|
|
# mkcephfs -d /tmp/foo --init-local-daemons mds
|
|
# mkcephfs -d /tmp/foo --init-local-daemons osd
|
|
# mkcephfs -d /tmp/foo --prepare-mon
|
|
# mkcephfs -d /tmp/foo --init-local-daemons mon
|
|
# cp /tmp/foo/keyring.admin /etc/ceph/keyring
|
|
#
|
|
# or simply
|
|
#
|
|
# mkcephfs -a -c ceph.conf
|
|
#
|
|
|
|
set -e
|
|
|
|
# if we start up as ./mkcephfs, assume everything else is in the
|
|
# current directory too.
|
|
if [ `dirname $0` = "." ] && [ $PWD != "/etc/init.d" ]; then
|
|
BINDIR=.
|
|
LIBDIR=.
|
|
ETCDIR=.
|
|
else
|
|
BINDIR=@bindir@
|
|
LIBDIR=@libdir@/ceph
|
|
ETCDIR=@sysconfdir@/ceph
|
|
fi
|
|
|
|
usage_exit() {
|
|
echo "usage: $0 -a -c ceph.conf [-k adminkeyring] [--mkbtrfs]"
|
|
echo " to generate a new ceph cluster on all nodes; for advanced usage see man page"
|
|
echo " ** be careful, this WILL clobber old data; check your ceph.conf carefully **"
|
|
exit
|
|
}
|
|
|
|
. $LIBDIR/ceph_common.sh
|
|
|
|
|
|
allhosts=0
|
|
mkbtrfs=0
|
|
preparemonmap=0
|
|
prepareosdfs=""
|
|
initdaemon=""
|
|
initdaemons=""
|
|
preparemon=0
|
|
|
|
numosd=
|
|
useosdmap=
|
|
usecrushmapsrc=
|
|
usecrushmap=
|
|
verbose=0
|
|
adminkeyring=""
|
|
conf=""
|
|
dir=""
|
|
moreargs=""
|
|
auto_action=0
|
|
manual_action=0
|
|
|
|
while [ $# -ge 1 ]; do
|
|
case $1 in
|
|
-v )
|
|
verbose=1;
|
|
;;
|
|
--dir | -d)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
dir=$1
|
|
;;
|
|
--allhosts | -a)
|
|
allhosts=1
|
|
auto_action=1
|
|
;;
|
|
--prepare-monmap)
|
|
preparemonmap=1
|
|
manual_action=1
|
|
;;
|
|
--prepare-osdfs)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
prepareosdfs=$1
|
|
manual_action=1
|
|
;;
|
|
--init-daemon)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
initdaemon=$1
|
|
manual_action=1
|
|
;;
|
|
--init-local-daemons)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
initlocaldaemons=$1
|
|
manual_action=1
|
|
;;
|
|
--prepare-mon)
|
|
preparemon=1
|
|
manual_action=1
|
|
;;
|
|
--mkbtrfs)
|
|
mkbtrfs=1
|
|
;;
|
|
--conf | -c)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
conf=$1
|
|
;;
|
|
--numosd)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
numosd=$1
|
|
moreargs="$moreargs --numosd $1"
|
|
;;
|
|
--osdmap)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
useosdmap=$1
|
|
moreargs="$moreargs --osdmap $1"
|
|
;;
|
|
--crushmapsrc)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
usecrushmapsrc=$1
|
|
moreargs="$moreargs --crushmapsrc $1"
|
|
;;
|
|
--crushmap)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
usecrushmap=$1
|
|
moreargs="$moreargs --crushmap $1"
|
|
;;
|
|
-k)
|
|
[ -z "$2" ] && usage_exit
|
|
shift
|
|
adminkeyring=$1
|
|
;;
|
|
*)
|
|
echo unrecognized option \'$1\'
|
|
usage_exit
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
|
|
[ -z "$conf" ] && [ -n "$dir" ] && conf="$dir/conf"
|
|
|
|
if [ $manual_action -eq 0 ]; then
|
|
if [ $auto_action -eq 0 ]; then
|
|
echo "You must specify an action. See man page."
|
|
usage_exit
|
|
fi
|
|
elif [ $auto_action -eq 1 ]; then
|
|
echo "The -a option cannot be combined with other subcommands; see man page."
|
|
usage_exit
|
|
fi
|
|
|
|
### prepare-monmap ###
|
|
|
|
if [ $preparemonmap -eq 1 ]; then
|
|
echo "preparing monmap in $dir/monmap"
|
|
|
|
# first, make a list of monitors
|
|
mons=`$CCONF -c $conf -l mon | egrep -v '^mon$' | sort`
|
|
args=""
|
|
|
|
type="mon"
|
|
for name in $mons; do
|
|
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
|
|
get_conf addr "" "mon addr"
|
|
if [ -z "$addr" ]; then
|
|
echo "$0: monitor $name has no address defined." 1>&2
|
|
exit 1
|
|
fi
|
|
args=$args" --add $id $addr"
|
|
done
|
|
|
|
if [ -z "$args" ]; then
|
|
echo "$0: no monitors found in config, aborting." 1>&2
|
|
exit 1
|
|
fi
|
|
|
|
# build monmap
|
|
monmap="$dir/monmap"
|
|
echo $BINDIR/monmaptool --create --clobber $args --print $monmap || exit 1
|
|
$BINDIR/monmaptool --create --clobber $args --print $monmap || exit 1
|
|
|
|
# copy conf
|
|
cp $conf $dir/conf
|
|
|
|
exit 0
|
|
fi
|
|
|
|
|
|
### init-daemon ###
|
|
|
|
create_private_key()
|
|
{
|
|
get_conf keyring "$dir/keyring.$name" "keyring"
|
|
echo "creating private key for $name keyring $keyring"
|
|
$BINDIR/cauthtool --create-keyring --gen-key -n $name $keyring
|
|
$BINDIR/cauthtool -p -n $name $keyring > $dir/key.$name
|
|
}
|
|
|
|
if [ -n "$initdaemon" ]; then
|
|
name=$initdaemon
|
|
type=`echo $name | cut -c 1-3` # e.g. 'mon', if $name is 'mon1'
|
|
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
|
|
name="$type.$id"
|
|
|
|
if [ $type = "osd" ]; then
|
|
$BINDIR/cosd -c $conf --monmap $dir/monmap -i $id --mkfs
|
|
create_private_key
|
|
fi
|
|
|
|
if [ $type = "mds" ]; then
|
|
create_private_key
|
|
fi
|
|
|
|
if [ $type = "mon" ]; then
|
|
$BINDIR/cmon -c $conf --mkfs -i $id --monmap $dir/monmap --osdmap $dir/osdmap -k $dir/keyring.mon
|
|
fi
|
|
|
|
exit 0
|
|
fi
|
|
|
|
|
|
## init-local-daemons ##
|
|
|
|
if [ -n "$initlocaldaemons" ]; then
|
|
get_name_list "$initlocaldaemons"
|
|
for name in $what; do
|
|
type=`echo $name | cut -c 1-3` # e.g. 'mon', if $name is 'mon1'
|
|
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
|
|
num=$id
|
|
name="$type.$id"
|
|
|
|
check_host || continue
|
|
|
|
$0 -d $dir --init-daemon $name
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
|
|
### prepare-osdfs ###
|
|
|
|
if [ -n "$prepareosdfs" ]; then
|
|
name=$prepareosdfs
|
|
type=`echo $name | cut -c 1-3` # e.g. 'mon', if $name is 'mon1'
|
|
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
|
|
name="$type.$id"
|
|
|
|
get_conf osd_data "" "osd data"
|
|
get_conf osd_journal "" "osd journal"
|
|
get_conf btrfs_path "$osd_data" "btrfs path" # mount point defaults so osd data
|
|
get_conf btrfs_devs "" "btrfs devs"
|
|
first_dev=`echo $btrfs_devs | cut '-d ' -f 1`
|
|
get_conf btrfs_opt "noatime" "btrfs options"
|
|
[ -n "$btrfs_opt" ] && btrfs_opt="-o $btrfs_opt"
|
|
get_conf osd_user "root" "user"
|
|
|
|
if [ -n "$osd_journal" ] && echo "$btrfs_devs" | grep -q -w "$osd_journal" ; then
|
|
echo "ERROR: osd journal device ($osd_journal) also used by btrfs devs ($btrfs_devs)"
|
|
exit 1
|
|
fi
|
|
|
|
test -d $osd_data || mkdir -p $osd_data
|
|
|
|
if [ -n "$osd_journal" ]; then
|
|
test -d $osd_journal || mkdir -p `dirname $osd_journal`
|
|
fi
|
|
|
|
umount $btrfs_path || true
|
|
for f in $btrfs_devs ; do
|
|
umount $f || true
|
|
done
|
|
|
|
modprobe btrfs || true
|
|
mkfs.btrfs $btrfs_devs
|
|
btrfsctl -a
|
|
mount -t btrfs $btrfs_opt $first_dev $btrfs_path
|
|
chown $osd_user $btrfs_path
|
|
chmod +w $btrfs_path
|
|
|
|
exit 0
|
|
fi
|
|
|
|
|
|
|
|
### prepare-mon ###
|
|
|
|
if [ $preparemon -eq 1 ]; then
|
|
|
|
if [ -n "$useosdmap" ]; then
|
|
echo "Using osdmap $useosdmap"
|
|
cp $useosdmap $dir/osdmap
|
|
else
|
|
# build osdmap
|
|
echo "Building generic osdmap"
|
|
|
|
# find highest osd id - assumes the conf file complies with osd numbering requirements
|
|
maxosd=`$CCONF -c $conf -l osd | grep -v ^osd\$ | cut -c 5- | sort -n | tail -1`
|
|
echo " highest numbered osd in $conf is osd.$maxosd"
|
|
lastosd=$(($maxosd + 1))
|
|
|
|
if [ -z "$numosd" ]; then
|
|
get_conf numosd "$lastosd" "num osd" mon global
|
|
fi
|
|
echo " num osd = $numosd"
|
|
|
|
$BINDIR/osdmaptool --createsimple $numosd $dir/osdmap -c $conf
|
|
fi
|
|
|
|
# import crush map?
|
|
get_conf crushmapsrc "" "crush map src" mon global
|
|
if [ -n "$crushmapsrc" ]; then
|
|
echo Compiling crush map from $crushmapsrc to $dir/crushmap
|
|
$BINDIR/crushtool -c $crushmapsrc -o $dir/crushmap
|
|
fi
|
|
get_conf crushmap "$usecrushmap" "crush map" mon global
|
|
if [ -n "$crushmap" ]; then
|
|
echo Importing crush map from $crushmap
|
|
$BINDIR/osdmaptool --import-crush $crushmap $dir/osdmap
|
|
fi
|
|
|
|
# admin keyring
|
|
echo Generating admin key at $dir/keyring.admin
|
|
$BINDIR/cauthtool --create-keyring --gen-key -n client.admin $dir/keyring.admin
|
|
|
|
# mon keyring
|
|
echo Building initial monitor keyring
|
|
cp $dir/keyring.admin $dir/keyring.mon
|
|
$BINDIR/cauthtool -n client.admin --set-uid=0 \
|
|
--cap mon 'allow *' \
|
|
--cap osd 'allow *' \
|
|
--cap mds 'allow' \
|
|
$dir/keyring.mon
|
|
|
|
$BINDIR/cauthtool --gen-key -n mon. $dir/keyring.mon
|
|
|
|
for k in $dir/key.*
|
|
do
|
|
kname=`echo $k | sed 's/.*key\.//'`
|
|
ktype=`echo $kname | cut -c 1-3`
|
|
kid=`echo $kname | cut -c 4- | sed 's/^\\.//'`
|
|
kname="$ktype.$kid"
|
|
secret=`cat $k`
|
|
if [ "$ktype" = "osd" ]; then
|
|
$BINDIR/cauthtool -n $kname --add-key $secret $dir/keyring.mon \
|
|
--cap mon 'allow rwx' \
|
|
--cap osd 'allow *'
|
|
fi
|
|
if [ "$ktype" = "mds" ]; then
|
|
$BINDIR/cauthtool -n $kname --add-key $secret $dir/keyring.mon \
|
|
--cap mon "allow rwx" \
|
|
--cap osd 'allow *' \
|
|
--cap mds 'allow'
|
|
fi
|
|
done
|
|
|
|
exit 0
|
|
fi
|
|
|
|
|
|
|
|
|
|
|
|
### do everything via ssh ###
|
|
|
|
if [ $allhosts -eq 1 ]; then
|
|
|
|
verify_conf
|
|
|
|
# do it all
|
|
if [ -z "$dir" ]; then
|
|
dir=`mktemp -d -t mkcephfs.XXXXXXXXXX` || exit 1
|
|
echo "temp dir is $dir"
|
|
trap "rm -rf $dir ; exit" INT TERM EXIT
|
|
fi
|
|
|
|
$0 --prepare-monmap -d $dir -c $conf
|
|
|
|
# osd, mds
|
|
get_name_list "osd mds"
|
|
for name in $what; do
|
|
type=`echo $name | cut -c 1-3` # e.g. 'mon', if $name is 'mon1'
|
|
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
|
|
num=$id
|
|
name="$type.$id"
|
|
|
|
check_host || continue
|
|
|
|
if [ -n "$ssh" ]; then
|
|
rdir="/tmp/mkfs.ceph.$$"
|
|
echo pushing conf and monmap to $host:$rdir
|
|
do_cmd "mkdir -p $rdir"
|
|
scp -q $dir/conf $host:$rdir
|
|
scp -q $dir/monmap $host:$rdir
|
|
else
|
|
rdir=$dir
|
|
fi
|
|
|
|
if [ $mkbtrfs -eq 1 ] && [ "$type" = "osd" ]; then
|
|
do_root_cmd "$0 -d $rdir --prepare-osdfs $name"
|
|
fi
|
|
|
|
do_cmd "$0 -d $rdir --init-daemon $name"
|
|
|
|
# collect the key
|
|
if [ -n "$ssh" ]; then
|
|
echo collecting $name key
|
|
scp -q $host:$rdir/key.$name $dir
|
|
do_cmd "rm -r $rdir"
|
|
fi
|
|
done
|
|
|
|
# prepare monitors
|
|
$0 -d $dir --prepare-mon $moreargs
|
|
|
|
# mons
|
|
get_name_list "mon"
|
|
for name in $what; do
|
|
type=`echo $name | cut -c 1-3` # e.g. 'mon', if $name is 'mon1'
|
|
id=`echo $name | cut -c 4- | sed 's/^\\.//'`
|
|
num=$id
|
|
name="$type.$id"
|
|
|
|
check_host || continue
|
|
|
|
if [ -n "$ssh" ]; then
|
|
echo pushing everything to $host
|
|
ssh $host mkdir -p $rdir
|
|
scp -q $dir/* $host:$rdir
|
|
else
|
|
rdir=$dir
|
|
fi
|
|
|
|
do_cmd "$0 -d $rdir --init-daemon $name"
|
|
done
|
|
|
|
# admin keyring
|
|
if [ -z "$adminkeyring" ]; then
|
|
get_conf adminkeyring "/etc/ceph/keyring" "keyring" global
|
|
fi
|
|
echo "placing client.admin keyring in $adminkeyring"
|
|
cp $dir/keyring.admin $adminkeyring
|
|
|
|
exit 0
|
|
fi
|
|
|