mirror of https://github.com/schoebel/mars
515 lines
16 KiB
Bash
515 lines
16 KiB
Bash
|
#!/bin/bash
|
||
|
|
#
|
||
|
|
# This file is part of MARS project: http://schoebel.github.io/mars/
|
||
|
|
#
|
||
|
|
# Copyright (C) 2015 Thomas Schoebel-Theuer
|
||
|
|
# Copyright (C) 2015 1&1 Internet AG
|
||
|
|
#
|
||
|
|
# This program is free software; you can redistribute it and/or modify
|
||
|
|
# it under the terms of the GNU General Public License as published by
|
||
|
|
# the Free Software Foundation; either version 2 of the License, or
|
||
|
|
# (at your option) any later version.
|
||
|
|
#
|
||
|
|
# This program is distributed in the hope that it will be useful,
|
||
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||
|
|
# GNU General Public License for more details.
|
||
|
|
#
|
||
|
|
# You should have received a copy of the GNU General Public License along
|
||
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||
|
|
|
||
|
|
############################################################
|
||
|
|
|
||
|
|
# TST spring 2015 lab prototype for mass rollout of MARS
|
||
|
|
|
||
|
|
# Environment-specific actions are encoded into variables.
|
||
|
|
# Change them (e.g. in /etc/mars/rollout.conf) for adaptation to
|
||
|
|
# any other operating environment.
|
||
|
|
#
|
||
|
|
# A few conventions are firmly built in: resource names and LVM disk names
|
||
|
|
# must be equal. In addition, it is advisable that VM names and
|
||
|
|
# resource names should be also strongly related (but VM names
|
||
|
|
# may have suffixes like infong4711.schlund.de).
|
||
|
|
#
|
||
|
|
# Please feel free to adapt this to your needs.
|
||
|
|
|
||
|
|
set -o pipefail
|
||
|
|
|
||
|
|
orig_vars="$(set | grep '^[_A-Za-z0-9]\+=' | cut -d= -f1)"
|
||
|
|
|
||
|
|
# Defaults for configuration variables
|
||
|
|
default_config=${default_config:-/etc/mars/rollout.conf}
|
||
|
|
# The rest is hardcoded here in case the config file does not exist
|
||
|
|
dry_run=${dry_run:-0}
|
||
|
|
verbose=${verbose:-0}
|
||
|
|
confirm=${confirm:-1}
|
||
|
|
help=${help:-0}
|
||
|
|
phase="{0..8}"
|
||
|
|
use_fake_sync=${use_fake_sync:-1}
|
||
|
|
override_fake_sync=${override_fake_sync:-0}
|
||
|
|
fakeable_resources="${fakeable_resources:-}"
|
||
|
|
sshopt="${sshopt:--4 -A -T -o StrictHostKeyChecking=no -o ForwardX11=no -o KbdInteractiveAuthentication=no -o VerifyHostKeyDNS=no}"
|
||
|
|
primary="${primary:-}"
|
||
|
|
secondary="${secondary:-}"
|
||
|
|
devices="${devices:-}"
|
||
|
|
device_pattern="${device_pattern:-/dev/vg*/{infong,ovz\}*}"
|
||
|
|
device_remove_regex="${device_remove_regex:-.-md\|old\|-bak}"
|
||
|
|
lvcreate_cmd="${lvcreate_cmd:-lvcreate -I 4M -L512G -n mars}"
|
||
|
|
drbd_force_unload="${drbd_force_unload:-0}"
|
||
|
|
drbd_dstate_cmd="${drbd_dstate_cmd:-drbdadm dstate}"
|
||
|
|
drbd_dstate_pattern="${drbd_dstate_pattern:-UpToDate/UpToDate}"
|
||
|
|
drbd_get_resources="${drbd_get_resources:-configure_InfongSpace.pl --list all | awk '{ print \$1; }' | sort -u}"
|
||
|
|
drbd_down_cmd="${drbd_down_cmd:-drbdadm down all || echo IGNORING failed DRBD shutdown because the kernel module will be unloaded anyway}"
|
||
|
|
drbd_update_config_res="${drbd_update_config_res:-configure_InfongSpace.pl --update-infong \$res repltype=mars}"
|
||
|
|
drbd_update_config_global="${drbd_update_config_global:-configure_InfongSpace.pl --write-drbd-conf}"
|
||
|
|
drbd_stop_cmd="${drbd_stop_cmd:-/etc/init.d/drbd stop || { ! [[ -e /proc/drbd ]] && echo stopping DRBD by hand && rmmod drbd; \}}"
|
||
|
|
mars_start_cmd="${mars_start_cmd:-ui-config-modify -c MARS_ENABLED=true; /etc/init.d/mars start}"
|
||
|
|
vm_reinit_cmd="${vm_reinit_cmd:-/etc/init.d/clustermanager restart; sleep 20; cm3 --stop all; sleep 5; cm3 -us}"
|
||
|
|
vm_status_cmd="${vm_status_cmd:-cm3 -us}"
|
||
|
|
vm_stopped_all_cmd="${vm_stopped_all_cmd:-cm3 --list-vms --with-status | grep -v '^\$' | grep -vi stopped | grep '.'}"
|
||
|
|
vm_stop_cmd="${vm_stop_cmd:-cm3 --stop all || { sleep 10; /etc/init.d/clustermanager restart && sleep 20 && cm3 --stop all; \}}"
|
||
|
|
vm_start_cmd="${vm_start_cmd:-/etc/init.d/clustermanager restart; sleep 20; cm3 --stop all; /etc/init.d/clustermanager restart; sleep 20; cm3 --stop all; cm3 -us; cm3 --start all; sleep 10; cm3 -us; for dummy in {0..2\}; do count=0; for i in \$(cm3 --list-vms --with-status | grep -i broken | cut -d: -f1); do echo \"RESTARTING BROKEN \$i\"; (( count++ )); cm3 -us; sleep 10; cm3 --stop \$i; done; if (( count )); then sleep 10; cm3 --start all; sleep 10; fi; done}"
|
||
|
|
# END configuration variables
|
||
|
|
|
||
|
|
param_vars="$(set | grep '^[_A-Za-z0-9]\+=' | cut -d= -f1)"
|
||
|
|
|
||
|
|
function fail
|
||
|
|
{
|
||
|
|
local txt="${1:-Unkown failure}"
|
||
|
|
echo "FAILURE: $txt" >> /dev/stderr
|
||
|
|
exit -1
|
||
|
|
}
|
||
|
|
|
||
|
|
function do_confirm
|
||
|
|
{
|
||
|
|
local skip="$1"
|
||
|
|
local response
|
||
|
|
|
||
|
|
(( !confirm )) && return 0
|
||
|
|
|
||
|
|
[[ "$skip" != "" ]] && skip="S to skip, "
|
||
|
|
echo -n "[CONFIRM: Press ${skip}Return to continue, ^C to abort] "
|
||
|
|
read response
|
||
|
|
! [[ "$response" =~ ^[sS] ]]
|
||
|
|
return $?
|
||
|
|
}
|
||
|
|
|
||
|
|
function remote
|
||
|
|
{
|
||
|
|
local host="$1"
|
||
|
|
local cmd="$2"
|
||
|
|
local nofail="${3:-0}"
|
||
|
|
|
||
|
|
(( verbose > 0 )) && echo "Executing on $host: '$cmd'" >> /dev/stderr
|
||
|
|
[[ "${cmd## }" = "" ]] && return 0
|
||
|
|
if ssh $sshopt root@$host "$cmd"; then
|
||
|
|
return 0
|
||
|
|
elif (( nofail )); then
|
||
|
|
return $?
|
||
|
|
else
|
||
|
|
fail "ssh to '$host' command '$cmd' failed with status $?"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
function remote_action
|
||
|
|
{
|
||
|
|
local host="$1"
|
||
|
|
local cmd="$2"
|
||
|
|
|
||
|
|
if (( dry_run )); then
|
||
|
|
echo "DRY_RUN REMOTE $host ACTION '$cmd'"
|
||
|
|
elif (( confirm )); then
|
||
|
|
echo "REMOTE $host ACTION '$cmd'"
|
||
|
|
if do_confirm 1; then
|
||
|
|
remote "$host" "$cmd"
|
||
|
|
else
|
||
|
|
echo "SKIPPING $host ACTION '$cmd'"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
remote "$host" "$cmd"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
function _get_resource
|
||
|
|
{
|
||
|
|
local device="${1:-$(fail "Resource argument is missing")}" || exit $?
|
||
|
|
echo "$device" | sed 's:^.*/::'
|
||
|
|
}
|
||
|
|
|
||
|
|
function are_all_vms_stopped
|
||
|
|
{
|
||
|
|
local host="$1"
|
||
|
|
local ret=$(remote $host "{ $vm_stopped_all_cmd ; } 1>&2 ; echo \$?")
|
||
|
|
echo "VMs on $host are $( (( !ret )) && echo "NOT ")stopped" >> /dev/stderr
|
||
|
|
return $(( !ret ))
|
||
|
|
}
|
||
|
|
|
||
|
|
function source_when_possible
|
||
|
|
{
|
||
|
|
local file="$1"
|
||
|
|
local type="$2"
|
||
|
|
|
||
|
|
if [[ -r "$file" ]]; then
|
||
|
|
. "$file" || fail "$type file $file is not parsable"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
source_when_possible "$default_config" "config"
|
||
|
|
|
||
|
|
# Allow forceful override of any _known_ variable at the command line
|
||
|
|
for i; do
|
||
|
|
if [[ "$i" =~ ^--[-_A-Za-z0-9]+$ ]]; then
|
||
|
|
param="${i#--}"
|
||
|
|
var="${param//-/_}"
|
||
|
|
[[ "$(eval "echo \"\$$var\"")" = "" ]] && abort "Variable '$var' is unknown"
|
||
|
|
eval "$var=1"
|
||
|
|
elif [[ "$i" =~ ^--[-_A-Za-z0-9]+= ]]; then
|
||
|
|
param="${i#--}"
|
||
|
|
var="${param%%=*}"
|
||
|
|
var="${var//-/_}"
|
||
|
|
val="${param#*=}"
|
||
|
|
[[ "$(eval "echo \"\$$var\"")" = "" ]] && abort "Variable '$var' is unknown"
|
||
|
|
eval "$var=$val"
|
||
|
|
elif [[ "$i" =~ ^-h$ ]]; then
|
||
|
|
help=1
|
||
|
|
elif [[ "$i" =~ ^-v$ ]]; then
|
||
|
|
(( verbose++ ))
|
||
|
|
elif [[ "$primary" = "" ]]; then
|
||
|
|
primary="$i"
|
||
|
|
elif [[ "$secondary" = "" ]]; then
|
||
|
|
secondary="$i"
|
||
|
|
else
|
||
|
|
abort "bad parameter syntax '$i'"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
function do_help
|
||
|
|
{
|
||
|
|
cat <<EOF
|
||
|
|
usage: $0 [options] <primaryhost> <secondaryhost>
|
||
|
|
|
||
|
|
The following parameter variables can be either passed by the
|
||
|
|
environment, or used for hard overriding on the command line
|
||
|
|
via --variable=value syntax:
|
||
|
|
|
||
|
|
$(
|
||
|
|
declare -A orig
|
||
|
|
for i in $orig_vars; do
|
||
|
|
orig[$i]=1
|
||
|
|
done
|
||
|
|
for i in $param_vars; do
|
||
|
|
[[ "$i" =~ _vars$ ]] && continue
|
||
|
|
if (( !orig[$i] )); then
|
||
|
|
if [[ "$(eval "echo \${$i}")" =~ ^[0-9]+$ ]]; then
|
||
|
|
echo "$i=$(eval "echo \${$i}")"
|
||
|
|
else
|
||
|
|
echo "$i=\"$(eval "echo \${$i}")\""
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
)
|
||
|
|
EOF
|
||
|
|
}
|
||
|
|
|
||
|
|
if (( help )); then
|
||
|
|
do_help
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ "$primary" = "" ]]; then
|
||
|
|
do_help
|
||
|
|
fail "No primary hostname given"
|
||
|
|
fi
|
||
|
|
if [[ "$secondary" = "" ]]; then
|
||
|
|
do_help
|
||
|
|
fail "No secondary hostname given"
|
||
|
|
fi
|
||
|
|
[[ "$primary" = "$secondary" ]] && fail "Primary and secondary hostnames must be distinct"
|
||
|
|
|
||
|
|
function do_phase
|
||
|
|
{
|
||
|
|
local phase="$1"
|
||
|
|
local host
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "------- Phase $phase"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
case "$phase" in
|
||
|
|
0)
|
||
|
|
echo "Create the /mars filesystem when necessary, ensure that it is mounted"
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
if (( $(remote $host "ls /dev/*/mars 1>&2; echo \$?") )); then
|
||
|
|
local line="$(remote $host "vgdisplay -c | sort -n -t: -k16 -r | head -1")" || fail "Cannot determine VG"
|
||
|
|
local vg_name="$(echo "$line" | cut -d: -f1)"
|
||
|
|
[[ "${vg_name// /}" = "" ]] && fail "Invalid VG name '$vg_name'"
|
||
|
|
local pv_count="$(echo "$line" | cut -d: -f10)"
|
||
|
|
(( pv_count < 1 )) && fail "Invalid PV count '$pv_count'"
|
||
|
|
echo "Host $host VG '$vg_name' (has $pv_count physical volumes)"
|
||
|
|
remote_action $host "$lvcreate_cmd -i $pv_count $vg_name"
|
||
|
|
sleep 2
|
||
|
|
if (( $(remote $host "ls /dev/*/mars 1>&2; echo \$?") )); then
|
||
|
|
fail "No LV for /mars exists on $host"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
if (( $(remote $host "grep -q /mars /proc/mounts; echo \$?") )); then
|
||
|
|
remote_action $host "[[ -d /mars ]] || mkdir /mars; mount /mars || { mkfs.ext4 -L mars /dev/*/mars && mount /dev/*/mars /mars; }"
|
||
|
|
if (( $(remote $host "grep -q /mars /proc/mounts; echo \$?") )); then
|
||
|
|
fail "No /mars is mounted on $host"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
;;
|
||
|
|
|
||
|
|
1)
|
||
|
|
echo "Create/join the MARS cluster when necessary"
|
||
|
|
if (( $(remote $primary "ls -l /mars/uuid 1>&2; echo \$?") )); then
|
||
|
|
echo "Host $primary create-cluster"
|
||
|
|
remote_action $primary "marsadm create-cluster"
|
||
|
|
fi
|
||
|
|
if (( $(remote $secondary "ls -l /mars/uuid 1>&2; echo \$?") )); then
|
||
|
|
echo "Host $secondary join-cluster"
|
||
|
|
remote_action $secondary "marsadm join-cluster $primary"
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
|
||
|
|
2)
|
||
|
|
echo "Stop VMs when necessary"
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
if are_all_vms_stopped $host; then
|
||
|
|
echo "No VMs are running on host $host."
|
||
|
|
else
|
||
|
|
echo "Some VMs are running on host $host"
|
||
|
|
(( !downtime_start )) && downtime_start=$(date +%s)
|
||
|
|
remote_action $host "$vm_stop_cmd"
|
||
|
|
downtime_end=$(date +%s)
|
||
|
|
echo "ESTIMATED operation duration: $(( downtime_end - downtime_start )) seconds"
|
||
|
|
if ! are_all_vms_stopped $host; then
|
||
|
|
fail "Some VMs are running on host $host"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
if (( downtime_start )); then
|
||
|
|
echo "ESTIMATED total shutdown operation duration: $(( downtime_end - downtime_start )) seconds"
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
|
||
|
|
3)
|
||
|
|
echo "Stop DRBD when necessary"
|
||
|
|
if (( drbd_force_unload || !$(remote $primary "[[ -e /proc/drbd ]]; echo \$?") )); then
|
||
|
|
local drbd_res="$(remote $primary "$drbd_get_resources")" || fail "Cannot get DRBD resources on $primary"
|
||
|
|
echo "DRBD resources on host $primary: $(echo $drbd_res)"
|
||
|
|
local cmd="for i in $(echo $drbd_res); do echo -n \"\$i \"; $drbd_dstate_cmd \$i; done"
|
||
|
|
echo "DRBD dstate on host $primary:"
|
||
|
|
local tmpfile=/tmp/dstate.$primary.$$
|
||
|
|
remote $primary "$cmd" | tee $tmpfile
|
||
|
|
if grep -qv "$drbd_dstate_pattern" < $tmpfile; then
|
||
|
|
echo "DRBD on $primary is NOT in sync"
|
||
|
|
else
|
||
|
|
echo "DRBD on $primary is in sync"
|
||
|
|
fi
|
||
|
|
if (( use_fake_sync )); then
|
||
|
|
echo "The following resources are fakeable:"
|
||
|
|
while read res txt; do
|
||
|
|
echo "$res $txt"
|
||
|
|
fakeable_resources+=" $res"
|
||
|
|
done <<EOF
|
||
|
|
$(grep "$drbd_dstate_pattern" < $tmpfile)
|
||
|
|
EOF
|
||
|
|
echo "List of fakeable DRBD resources: $fakeable_resources"
|
||
|
|
fi
|
||
|
|
rm -f $tmpfile
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
echo "Creating DRBD backup on $host"
|
||
|
|
remote_action $host "tar czvf /var/backups/drbd-config-$(date +%Y%m%d-%H%M).tgz /etc/drbd* || true"
|
||
|
|
echo "Shutdown DRBD on $host"
|
||
|
|
remote_action $host "$drbd_down_cmd"
|
||
|
|
local res
|
||
|
|
local cmd=""
|
||
|
|
for res in $drbd_res; do
|
||
|
|
cmd+="${drbd_update_config_res/\$res/$res} ; "
|
||
|
|
done
|
||
|
|
cmd+="$drbd_update_config_global ; $drbd_stop_cmd"
|
||
|
|
remote_action $host "$cmd"
|
||
|
|
done
|
||
|
|
else
|
||
|
|
echo "DRBD is NOT in use, switching off fake-sync"
|
||
|
|
use_fake_sync=0
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
|
||
|
|
4)
|
||
|
|
echo "Start MARS when necessary"
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
if (( $(remote $host "[[ -d /proc/sys/mars ]]; echo \$?") )); then
|
||
|
|
remote_action $host "$mars_start_cmd"
|
||
|
|
sleep 3 &
|
||
|
|
else
|
||
|
|
echo "MARS is already running on $host"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
wait
|
||
|
|
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
local device
|
||
|
|
local cmd=""
|
||
|
|
for device in $(eval "echo \${devices_${host//-/_}}"); do
|
||
|
|
local res="$(_get_resource $device)"
|
||
|
|
[[ "$res" = "" ]] && fail "Implausible resource name '$res'"
|
||
|
|
local this_size=${sizes[$res]}
|
||
|
|
(( this_size < 4096 )) && fail "Implausible device size '$this_size'"
|
||
|
|
if (( $(remote $host "[[ -e /mars/resource-$res/data-$host ]]; echo \$?") )); then
|
||
|
|
echo "RESOURCE $res on $host: device $device size $this_size"
|
||
|
|
if [[ "$host" = "$primary" ]]; then
|
||
|
|
cmd+="marsadm create-resource $res $device $res $this_size && "
|
||
|
|
else
|
||
|
|
[[ "$cmd" = "" ]] && cmd="marsadm wait-cluster ; "
|
||
|
|
cmd+="marsadm join-resource $res $device && "
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
echo "RESOURCE $res already exists on $host"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
if [[ "$cmd" != "" ]]; then
|
||
|
|
remote_action $host "$cmd true"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
if (( use_fake_sync )) && [[ "$fakeable_resources" != "" ]]; then
|
||
|
|
echo "Starting FAKE-SYNC on resources $fakeable_resources"
|
||
|
|
remote_action $secondary "for i in $fakeable_resources; do marsadm fake-sync \$i; done"
|
||
|
|
elif (( override_fake_sync )); then
|
||
|
|
echo "OVERRIDING FAKE-SYNC on ALL resources"
|
||
|
|
remote_action $secondary "marsadm fake-sync all"
|
||
|
|
else
|
||
|
|
echo "no fake-sync is executed"
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
|
||
|
|
5)
|
||
|
|
echo "Show status of MARS"
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
echo ""
|
||
|
|
echo "MARS Status on $host:"
|
||
|
|
remote $host "marsadm view all"
|
||
|
|
done
|
||
|
|
;;
|
||
|
|
|
||
|
|
6)
|
||
|
|
echo "Reinit VM clustermanager"
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
echo "------ Reinit $host:"
|
||
|
|
remote_action $host "$vm_reinit_cmd"
|
||
|
|
done
|
||
|
|
;;
|
||
|
|
|
||
|
|
7)
|
||
|
|
echo "Start VMs when necessary"
|
||
|
|
if are_all_vms_stopped $primary; then
|
||
|
|
uptime_start=$(date +%s)
|
||
|
|
remote_action $primary "$vm_start_cmd"
|
||
|
|
final=$(date +%s)
|
||
|
|
echo "ESTIMATED startup duration: $(( final - uptime_start )) seconds"
|
||
|
|
if (( downtime_start )); then
|
||
|
|
echo "ESTIMATED total VM downtime: $(( final - downtime_start )) seconds"
|
||
|
|
fi
|
||
|
|
echo ""
|
||
|
|
else
|
||
|
|
echo "Some VMs are running on host $primary. Please check by hand whether some of them need a restart."
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
|
||
|
|
8)
|
||
|
|
echo "Show status of VMs"
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
echo "------ Status on $host:"
|
||
|
|
remote $host "$vm_status_cmd"
|
||
|
|
done
|
||
|
|
;;
|
||
|
|
|
||
|
|
*)
|
||
|
|
echo "Unknown / unimplemented phase '$phase'"
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
function main
|
||
|
|
{
|
||
|
|
echo "Script $0 running phase $phase"
|
||
|
|
echo ""
|
||
|
|
echo "Params: $0 $*"
|
||
|
|
echo ""
|
||
|
|
echo "primary: '$primary'"
|
||
|
|
echo "secondary: '$secondary'"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
script_start=$(date +%s)
|
||
|
|
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
ping -c 1 $host || fail "Host '$primary' is not reachable"
|
||
|
|
remote $host uptime || fail "ssh connection to '$host' does not work. Ensure that ssh-agent is running."
|
||
|
|
done
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# when necessary, determine list of devices
|
||
|
|
|
||
|
|
if [[ "$devices" = "" ]]; then
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
eval "devices_${host//-/_}=\"$(remote $host "ls $device_pattern" 2>/dev/null | grep -v "$device_remove_regex")\"" || fail "cannot determine devices on $host"
|
||
|
|
eval "echo devices_${host//-/_}: \${devices_${host//-/_}}"
|
||
|
|
done
|
||
|
|
else
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
eval "devices_${host//-/_}=\"$devices\""
|
||
|
|
done
|
||
|
|
echo "Using given devices '$devices' for both hosts $primary $secondary"
|
||
|
|
fi
|
||
|
|
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
[[ "$(eval "echo \${devices_${host//-/_}}")" = "" ]] && fail "No devices have been determined on $host"
|
||
|
|
eval "resources_${host//-/_}=\"\$(for i in \${devices_${host//-/_}}; do _get_resource "\$i"; done | sort)\""
|
||
|
|
eval "echo resources_${host//-/_}: \${resources_${host//-/_}}"
|
||
|
|
[[ "$(eval "echo \${resources_${host//-/_}}")" = "" ]] && fail "No resources have been determined on $host"
|
||
|
|
done
|
||
|
|
if [[ "$(eval "echo \${resources_${primary//-/_}}")" != "$(eval "echo \${resources_${secondary//-/_}}")" ]]; then
|
||
|
|
fail "Primary resource list is different from secondary resource list"
|
||
|
|
fi
|
||
|
|
|
||
|
|
declare -A sizes
|
||
|
|
|
||
|
|
for host in $primary $secondary; do
|
||
|
|
echo "Host $host:"
|
||
|
|
while read device sector_size; do
|
||
|
|
this_size=$(( sector_size * 512 ))
|
||
|
|
echo " device $device: size $this_size"
|
||
|
|
this_resource="$(_get_resource $device)"
|
||
|
|
if (( !sizes[$this_resource] || this_size < sizes[$this_resource] )); then
|
||
|
|
sizes[$this_resource]=$this_size
|
||
|
|
fi
|
||
|
|
done <<EOF
|
||
|
|
$(remote $host "/sbin/lvdisplay -c $(eval "echo \${devices_${host//-/_}}") | cut -d: -f1,7" | sed 's/:/ /')
|
||
|
|
EOF
|
||
|
|
done
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "Determined the following sizes:"
|
||
|
|
for res in ${!sizes[*]}; do
|
||
|
|
echo " $res: ${sizes[$res]}"
|
||
|
|
done
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
do_confirm
|
||
|
|
|
||
|
|
for this_phase in $(eval "echo $phase"); do
|
||
|
|
do_phase $this_phase
|
||
|
|
done
|
||
|
|
|
||
|
|
script_end=$(date +%s)
|
||
|
|
echo "ESTIMATED script duration: $(( script_end - script_start )) seconds"
|
||
|
|
}
|
||
|
|
|
||
|
|
downtime_start=0
|
||
|
|
uptime_start=0
|
||
|
|
|
||
|
|
main 2>&1 | tee rollout-$(date +%Y%m%d-%H%M).$primary.$secondary.log
|