mirror of
https://github.com/schoebel/mars
synced 2024-12-12 01:36:06 +00:00
1439 lines
40 KiB
Bash
Executable File
1439 lines
40 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# This file is part of MARS project: http://schoebel.github.io/mars/
|
|
#
|
|
# Copyright (C) 2017 Thomas Schoebel-Theuer
|
|
# Copyright (C) 2017 1&1 Internet AG
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
############################################################
|
|
|
|
# TST summer 2017 lab prototype
|
|
|
|
# Generic MARS background migration of a VM / container.
|
|
# Plugins can be used for adaptation to system-specific sub-operations
|
|
# (e.g. the 1&1-specific clustermanager cm3)
|
|
|
|
# There are some basic conventions / assumptions:
|
|
# - MARS resource names are equal to LV names and to KVM / LXC names
|
|
# - All hosts are in DNS with their pure names (accessible via resolv.conf)
|
|
# - There is a 1:n relationship between each
|
|
# $storage_host : $hypervisor_host : $container_host
|
|
|
|
set -o pipefail
|
|
shopt -s nullglob
|
|
export LC_ALL=C
|
|
export start_stamp="$(date "+%F_%T" | sed 's/:/./g')"
|
|
|
|
# parameters
|
|
operation="${operation:-}"
|
|
res="${res:-}"
|
|
target_primary="${target_primary:-}"
|
|
target_secondary="${target_secondary:-}"
|
|
target_percent=${target_percent:-85}
|
|
|
|
# short options
|
|
dry_run=${dry_run:-0}
|
|
verbose=${verbose:-0}
|
|
confirm=${confirm:-1}
|
|
force=${force:-0}
|
|
logdir="${logdir:-.}"
|
|
min_space="${min_space:-20000000}"
|
|
|
|
# more complex options
|
|
ssh_opt="${ssh_opt:--4 -A -o StrictHostKeyChecking=no -o ForwardX11=no -o KbdInteractiveAuthentication=no -o VerifyHostKeyDNS=no}"
|
|
rsync_opt="${rsync_opt:- -aSH --info=STATS}"
|
|
rsync_opt_prepare="${rsync_opt_prepare:---exclude='.filemon2' --delete}"
|
|
rsync_opt_hot="${rsync_opt_hot:---delete}"
|
|
rsync_nice="${rsync_nice:-nice -19}"
|
|
rsync_repeat_prepare="${rsync_repeat_prepare:-5}"
|
|
rsync_repeat_hot="${rsync_repeat_hot:-3}"
|
|
|
|
lvremove_opt="${lvremove_opt:--f}"
|
|
|
|
# some constants
|
|
tmp_suffix="${tmp_suffix:--tmp}"
|
|
shrink_suffix_old="${shrink_suffix_old:--preshrink}"
|
|
commands_needed="${commands_needed:-ssh rsync grep sed awk sort head tail tee cat ls cut ping date mkdir rm bc}"
|
|
|
|
######################################################################
|
|
|
|
# help
|
|
|
|
function helpme
|
|
{
|
|
cat <<EOF
|
|
Usage:
|
|
$0 --help
|
|
Show help
|
|
$0 --variable=<value>
|
|
Override any shell variable
|
|
|
|
Actions for resource migration:
|
|
|
|
$0 migrate_prepare <resource> <target_primary> [<target_secondary>]
|
|
Allocate LVM space at the targets and start MARS replication.
|
|
|
|
$0 migrate_wait <resource> <target_primary> [<target_secondary>]
|
|
Wait until MARS replication reports UpToDate.
|
|
|
|
$0 migrate_finish <resource> <target_primary> [<target_secondary>]
|
|
Call hooks for handover to the targets.
|
|
|
|
$0 migrate <resource> <target_primary> [<target_secondary>]
|
|
Run the sequence migrate_prepare ; migrate_wait ; migrate_finish.
|
|
|
|
$0 migrate_cleanup <resource>
|
|
Remove old / currently unused LV replicas from MARS and deallocate from LVM.
|
|
|
|
$0 manual_migrate_config <resource> <target_primary> [<target_secondary>]
|
|
Transfer only the cluster config, without changing the MARS replicas.
|
|
This does no resource stopping / restarting.
|
|
Useful for reverting a failed migration.
|
|
|
|
$0 manual_config_update <hostname>
|
|
Only update the cluster config, without changing anything else.
|
|
Useful for manual repair of failed migration.
|
|
|
|
|
|
Actions for inplace FS shrinking:
|
|
|
|
$0 shrink_prepare <resource> [<percent>]
|
|
Allocate temporary LVM space (when possible) and create initial raw FS copy.
|
|
Default percent value(when left out) is $target_percent.
|
|
|
|
$0 shrink_finish <resource>
|
|
Incrementally update the FS copy, swap old <=> new copy with small downtime.
|
|
|
|
$0 shrink_cleanup <resource>
|
|
Remove old FS copy from LVM.
|
|
|
|
$0 shrink <resource> <percent>
|
|
Run the sequence shrink_prepare ; shrink_finish ; shrink_cleanup.
|
|
|
|
Actions for inplace FS extension:
|
|
|
|
$0 extend <resource> <percent>
|
|
|
|
Global maintenance:
|
|
|
|
$0 lv_cleanup <resource>
|
|
|
|
General features:
|
|
|
|
- instead of <percent>, an absolute amount of storage with suffix
|
|
'k' or 'm' or 'g' can be given.
|
|
|
|
- when <resource> is currently stopped, login to the container is
|
|
not possible, and in turn the hypervisor node and primary storage node
|
|
cannot be automatically determined. In such a case, the missing
|
|
nodes can be specified via the syntax
|
|
<resource>:<hypervisor>:<primary_storage>
|
|
|
|
- the following LV suffixes are used (naming convention):
|
|
$tmp_suffix = currently emerging version for shrinking
|
|
$shrink_suffix_old = old version before shrinking took place
|
|
EOF
|
|
source_hooks
|
|
verbose=0 call_hook hook_describe_plugin
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# basic infrastructure
|
|
|
|
function fail
|
|
{
|
|
local txt="${1:-Unkown failure}"
|
|
echo "FAILURE: $txt" >> /dev/stderr
|
|
exit -1
|
|
}
|
|
|
|
# Unfortunately, the bash has no primitive for running an arbitrary
|
|
# (complex) command until some timeout is exceeded.
|
|
#
|
|
# Workaround by disjoint waiting for an additional background sleep process.
|
|
#
|
|
function timeout_cmd
|
|
{
|
|
local cmd="$1"
|
|
local limit="${2:-30}"
|
|
local do_fail="${3:-0}"
|
|
|
|
if (( limit <= 0 )); then # timeout is disabled
|
|
bash -c "$cmd"
|
|
local rc=$?
|
|
#echo "RC=$rc" >> /dev/stderr
|
|
return $rc
|
|
fi
|
|
|
|
set +m
|
|
eval "$cmd" &
|
|
local cmd_pid=$!
|
|
|
|
sleep $limit &
|
|
local sleep_pid=$!
|
|
|
|
# disjoint waiting
|
|
wait -n $cmd_pid $sleep_pid
|
|
local rc1=$?
|
|
#echo "RC1=$rc1" >> /dev/stderr
|
|
|
|
kill $sleep_pid > /dev/null 2>&1
|
|
kill $cmd_pid > /dev/null 2>&1
|
|
wait $cmd_pid > /dev/null 2>&1
|
|
local rc2=$?
|
|
#echo "RC2=$rc2" >> /dev/stderr
|
|
|
|
# ensure to eat the background status, +m alone is not enough
|
|
wait $sleep_pid > /dev/null 2>&1
|
|
|
|
if (( rc2 == 143 )); then
|
|
if (( do_fail )); then
|
|
fail "TIMEOUT $limit seconds for '$cmd' reached"
|
|
else
|
|
echo "TIMEOUT $limit seconds for '$cmd' reached" >> /dev/stderr
|
|
fi
|
|
fi
|
|
|
|
local rc=$(( rc1 | rc2 ))
|
|
#echo "RC=$rc" >> /dev/stderr
|
|
return $rc
|
|
}
|
|
|
|
function source_hooks
|
|
{
|
|
local dir
|
|
local path
|
|
|
|
declare -g -A sourced_hook
|
|
for dir in /etc/mars/hooks ./hooks .; do
|
|
for path in $dir/hooks-*.sh; do
|
|
[[ "${sourced_hook[$path]}" != "" ]] && continue
|
|
echo "Sourcing hooks in '$path'"
|
|
source $path || fail "cannot source '$path'"
|
|
sourced_hook[$path]=1
|
|
done
|
|
done
|
|
}
|
|
|
|
args_info=""
|
|
|
|
function scan_args
|
|
{
|
|
local -a params
|
|
local index=0
|
|
local par
|
|
for par in "$@"; do
|
|
if [[ "$par" = "--help" ]]; then
|
|
helpme
|
|
exit 0
|
|
elif [[ "$par" =~ "=" ]]; then
|
|
par="${par#--}"
|
|
local lhs="$(echo "$par" | cut -d= -f1)"
|
|
local rhs="$(echo "$par" | cut -d= -f2-)"
|
|
lhs="${lhs//-/_}"
|
|
echo "$lhs=$rhs"
|
|
eval "$lhs=$rhs"
|
|
continue
|
|
elif [[ ":$par" =~ ":--" ]]; then
|
|
par="${par#--}"
|
|
par="${par//-/_}"
|
|
echo "$par=1"
|
|
eval "$par=1"
|
|
continue
|
|
fi
|
|
if (( !index )); then
|
|
if [[ "$par" =~ migrate_cleanup|lv_cleanup ]]; then
|
|
local -a params=(operation res)
|
|
elif [[ "$par" =~ shrink|extend ]]; then
|
|
local -a params=(operation res target_percent)
|
|
elif [[ "$par" =~ migrate ]]; then
|
|
local -a params=(operation res target_primary target_secondary)
|
|
elif [[ "$par" =~ manual_config_update ]]; then
|
|
local -a params=(operation host)
|
|
else
|
|
helpme
|
|
fail "unknown operation '$1'"
|
|
fi
|
|
fi
|
|
local lhs="${params[index]}"
|
|
if [[ "$lhs" != "" ]]; then
|
|
echo "$lhs=$par"
|
|
eval "$lhs=$par"
|
|
args_info+=".${par//:/_}"
|
|
(( index++ ))
|
|
else
|
|
helpme
|
|
fail "stray parameter '$par'"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function do_confirm
|
|
{
|
|
local skip="$1"
|
|
local response
|
|
|
|
(( !confirm )) && return 0
|
|
|
|
[[ "$skip" != "" ]] && skip="S to skip, "
|
|
echo -n "[CONFIRM: Press ${skip}Return to continue, ^C to abort] "
|
|
read -e response
|
|
! [[ "$response" =~ ^[sS] ]]
|
|
return $?
|
|
}
|
|
|
|
function remote
|
|
{
|
|
local host="$1"
|
|
local cmd="$2"
|
|
local nofail="${3:-0}"
|
|
|
|
(( verbose > 0 )) && echo "Executing on $host: '$cmd'" >> /dev/stderr
|
|
[[ "$host" = "" ]] && return
|
|
[[ "${cmd## }" = "" ]] && return
|
|
ssh $ssh_opt "root@$host" "$cmd"
|
|
local rc=$?
|
|
if (( !rc )); then
|
|
return 0
|
|
elif (( nofail )); then
|
|
return $rc
|
|
else
|
|
fail "ssh to '$host' command '$cmd' failed with status $rc"
|
|
fi
|
|
}
|
|
|
|
function remote_action
|
|
{
|
|
local host="$1"
|
|
local cmd="$2"
|
|
|
|
if (( dry_run )); then
|
|
echo "DRY_RUN REMOTE $host ACTION '$cmd'"
|
|
elif (( confirm )); then
|
|
echo "REMOTE $host ACTION '$cmd'"
|
|
if do_confirm 1; then
|
|
remote "$host" "$cmd"
|
|
else
|
|
echo "SKIPPING $host ACTION '$cmd'"
|
|
fi
|
|
else
|
|
remote "$host" "$cmd"
|
|
fi
|
|
}
|
|
|
|
function log
|
|
{
|
|
local dir="$1"
|
|
local file="$2"
|
|
|
|
if [[ "$dir" != "" ]] && [[ "$file" != "" ]]; then
|
|
tee -a "$dir/$file"
|
|
else
|
|
cat
|
|
fi
|
|
}
|
|
|
|
section_nr=1
|
|
|
|
function section
|
|
{
|
|
local txt="${1:--}"
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo "$(( section_nr++ )). $txt"
|
|
echo ""
|
|
}
|
|
|
|
function commands_installed
|
|
{
|
|
local cmd_list="$1"
|
|
|
|
local cmd
|
|
for cmd in $cmd_list; do
|
|
if ! which $cmd; then
|
|
fail "shell command '$cmd' is not installed"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function exists_hook
|
|
{
|
|
local name="$1"
|
|
[[ "$(type -t $name)" =~ function ]]
|
|
}
|
|
|
|
function call_hook
|
|
{
|
|
local name="$1"
|
|
if exists_hook "$name"; then
|
|
(( verbose )) && echo "Running hook: $name $@" >> /dev/stderr
|
|
shift
|
|
$name "$@" || fail "cannot execute hook function '$name'"
|
|
else
|
|
echo "Skipping undefined hook '$name'" >> /dev/stderr
|
|
fi
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# helper functions for determining hosts / relationships
|
|
|
|
declare -A hypervisor_host
|
|
|
|
function get_hyper
|
|
{
|
|
local res="$1"
|
|
|
|
declare -g hypervisor_host
|
|
local hyper="${hypervisor_host[$res]}"
|
|
if [[ "$hyper" = "" ]]; then
|
|
hyper="$(call_hook hook_get_hyper "$res")" ||\
|
|
fail "Cannot determine hypervisor hostname for resource '$res'"
|
|
hypervisor_host[$res]="$hyper"
|
|
fi
|
|
[[ "$hyper" = "" ]] && return -1
|
|
echo "$hyper"
|
|
}
|
|
|
|
declare -A storage_host
|
|
|
|
function get_store
|
|
{
|
|
local res="$1"
|
|
|
|
declare -g storage_host
|
|
local store="${storage_host[$res]}"
|
|
if [[ "$store" = "" ]]; then
|
|
store="$(call_hook hook_get_store "$res")" ||\
|
|
fail "Cannot determine storage hostname for resource '$res'"
|
|
if [[ "$store" = "" ]]; then
|
|
# assume local storage
|
|
store="$(get_hyper "$res")"
|
|
fi
|
|
storage_host[$res]="$store"
|
|
fi
|
|
[[ "$store" = "" ]] && return -1
|
|
echo "$store"
|
|
}
|
|
|
|
declare -A vgs
|
|
|
|
function get_vg
|
|
{
|
|
local host="$1"
|
|
|
|
declare -g vgs
|
|
local vg="${vgs[$host]}"
|
|
if [[ "$vg" = "" ]]; then
|
|
vg="$(call_hook hook_get_vg "$host")" ||\
|
|
fail "Cannot determine volume group for host '$host'"
|
|
vgs[$host]="$vg"
|
|
fi
|
|
[[ "$vg" = "" ]] && return -1
|
|
echo "$vg"
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# LV cleanup over the whole pool (may take some time)
|
|
|
|
function LV_cleanup
|
|
{
|
|
local primary="$1"
|
|
local lv_name="$2"
|
|
local do_it="${3:-0}"
|
|
|
|
local total_count=0
|
|
local remove_count=0
|
|
section "Determine hosts and LVs for cleanup"
|
|
|
|
local to_check="$(remote "$primary" "marsadm view-cluster-members")"
|
|
echo "Determined the following cluster members: " $to_check >> /dev/stderr
|
|
|
|
section "Run over the host list for cleanup"
|
|
|
|
echo "do_remove:host:LV_path"
|
|
local host
|
|
for host in $to_check; do
|
|
local path
|
|
for path in $(remote "$host" "ls /dev/*/$lv_name*" 2>/dev/null | grep -v "/mars/" ); do
|
|
local do_remove=0
|
|
local disk="$(remote "$host" "marsadm view-get-disk $lv_name")" 2>/dev/null
|
|
if [[ "$disk" = "" ]]; then
|
|
do_remove=1
|
|
(( remove_count++ ))
|
|
fi
|
|
echo "$do_remove:$host:$path"
|
|
(( total_count++ ))
|
|
if (( do_remove && do_it )); then
|
|
call_hook hook_disconnect "$host" "$lv_name"
|
|
remote "$host" "lvremove $lvremove_opt $path"
|
|
fi
|
|
done
|
|
done
|
|
echo "---------------"
|
|
echo "Total number of LVs: $total_count"
|
|
echo "Total number to remove: $remove_count"
|
|
if (( !do_it && !total )); then
|
|
echo "Nothing to do. Exiting."
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# checks for LV migration
|
|
|
|
function check_migration
|
|
{
|
|
# works on global parameters
|
|
[[ "$target_primary" = "" ]] && fail "target hostname is not defined"
|
|
[[ "$target_primary" = "$primary" ]] && fail "target host '$target_primary' needs to be distinct from source host"
|
|
for host in $target_primary $target_secondary; do
|
|
ping -c 1 "$host" > /dev/null || fail "Host '$host' is not pingable"
|
|
remote "$host" "mountpoint /mars > /dev/null"
|
|
remote "$host" "[[ -d /mars/ips/ ]]"
|
|
done
|
|
call_hook hook_check_host "$primary $secondary_list $target_primary $target_secondary"
|
|
}
|
|
|
|
function check_vg_space
|
|
{
|
|
local host="$1"
|
|
local min_size="$2"
|
|
|
|
[[ "$host" = "" ]] && return
|
|
|
|
local vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
local rest="$(remote "$host" "vgs --noheadings -o \"vg_free\" --units k $vg_name" | sed 's/\.[0-9]\+//' | sed 's/k//')" || fail "cannot determine VG rest space"
|
|
echo "$vg_name REST space on '$host' : $rest"
|
|
if (( rest <= min_size )); then
|
|
fail "NOT ENOUGH SPACE on $host (needed: $min_size)"
|
|
fi
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# actions for LV migration
|
|
|
|
function get_stripe_extra
|
|
{
|
|
# compute LVM stripe number
|
|
local stripes="$(remote "$host" "vgs" | grep '$vg_name ' | awk '{ print $2; }')"
|
|
local extra=""
|
|
if (( stripes > 1 )); then
|
|
echo "Using $stripes LVM stripes" >> /dev/stderr
|
|
extra="-i $stripes"
|
|
fi
|
|
echo "$extra"
|
|
}
|
|
|
|
function create_migration_space
|
|
{
|
|
local host="$1"
|
|
local lv_name="$2"
|
|
local size="$3"
|
|
|
|
# some checks
|
|
[[ "$host" = "" ]] && return
|
|
local vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
remote "$host" "if [[ -e /dev/$vg_name/${lv_name} ]]; then echo \"REFUSING to overwrite /dev/$vg_name/${lv_name} on $host - Do this by hand\"; exit -1; fi"
|
|
local extra="$(get_stripe_extra "$host" "$vg_name")"
|
|
|
|
# do it
|
|
remote "$host" "lvcreate -L ${size}k $etxra -n $lv_name $vg_name"
|
|
}
|
|
|
|
function migration_prepare
|
|
{
|
|
local source_primary="$1"
|
|
local lv_name="$2"
|
|
local target_primary="$3"
|
|
local target_secondary="$4"
|
|
|
|
section "Ensure that \"marsadm merge-cluster\" has been executed."
|
|
|
|
# This is idempotent.
|
|
if exists_hook hook_merge_cluster; then
|
|
call_hook hook_merge_cluster "$source_primary" "$target_primary"
|
|
call_hook hook_merge_cluster "$source_primary" "$target_secondary"
|
|
else
|
|
remote "$target_primary" "marsadm merge-cluster $source_primary"
|
|
remote "$target_secondary" "marsadm merge-cluster $source_primary"
|
|
fi
|
|
|
|
remote "$target_primary" "marsadm wait-cluster"
|
|
|
|
section "Idempotence: check whether the additional replica has been alread created"
|
|
|
|
local already_present="$(remote "$target_primary" "marsadm view-is-attach $lv_name")"
|
|
if (( already_present )); then
|
|
echo "Nothing to do: resource '$lv_name' is already present at '$target_primary'"
|
|
return
|
|
fi
|
|
|
|
section "Re-determine and check all resource sizes for safety"
|
|
|
|
local size="$(( $(remote "$source_primary" "marsadm view-sync-size $lv_name") / 1024 ))" ||\
|
|
fail "cannot determine resource size"
|
|
|
|
check_vg_space "$target_primary" "$size"
|
|
check_vg_space "$target_secondary" "$size"
|
|
|
|
local primary_vg_name="$(get_vg "$target_primary")"
|
|
local secondary_vg_name="$(get_vg "$target_secondary")"
|
|
local primary_dev="/dev/$primary_vg_name/${lv_name}"
|
|
local secondary_dev="/dev/$secondary_vg_name/${lv_name}"
|
|
|
|
section "Create migration spaces"
|
|
|
|
create_migration_space "$target_primary" "$lv_name" "$size"
|
|
create_migration_space "$target_secondary" "$lv_name" "$size"
|
|
|
|
section "Join the resources"
|
|
|
|
if exists_hook hook_join_resource; then
|
|
call_hook hook_join_resource "$source_primary" "$target_primary" "$lv_name" "$primary_dev"
|
|
call_hook hook_join_resource "$source_primary" "$target_secondary" "$lv_name" "$secondary_dev"
|
|
else
|
|
remote "$target_primary" "marsadm join-resource $lv_name $primary_dev"
|
|
remote "$target_secondary" "marsadm join-resource $lv_name $secondary_dev"
|
|
fi
|
|
remote "$target_primary" "marsadm wait-cluster"
|
|
}
|
|
|
|
function wait_resource_uptodate
|
|
{
|
|
local host_list="$1"
|
|
local res="$2"
|
|
|
|
section "Wait for MARS UpToDate"
|
|
|
|
local host
|
|
for host in $host_list; do
|
|
remote "$host" "marsadm wait-cluster"
|
|
done
|
|
(( verbose )) && echo "$(date) sync rests for '$host_list':"
|
|
local max_wait=15
|
|
while true; do
|
|
(( verbose )) && echo -n "$(date) sync rests:"
|
|
local syncing=0
|
|
local total_rest=0
|
|
for host in $host_list; do
|
|
local rest="$(verbose=0 remote "$host" "marsadm view-sync-rest $res")"
|
|
if (( verbose )); then
|
|
if (( rest < 1024 )); then
|
|
echo -n " $(( rest ))B"
|
|
elif (( rest < 1024 * 1024 )); then
|
|
echo -n " $(( rest / 1024 ))KiB"
|
|
elif (( rest < 1024 * 1024 * 1024 )); then
|
|
echo -n " $(( rest / 1024 / 1024 ))MiB"
|
|
else
|
|
echo -n " $(( rest / 1024 / 1024 / 1024 ))GiB"
|
|
fi
|
|
fi
|
|
if (( rest > 0 )); then
|
|
(( syncing++ ))
|
|
else
|
|
local status="$(verbose=0 remote "$host" "marsadm view-diskstate $res")"
|
|
(( verbose )) && echo -n "/$status"
|
|
if ! [[ "$status" =~ UpToDate ]]; then
|
|
(( syncing++ ))
|
|
fi
|
|
fi
|
|
(( total_rest += rest ))
|
|
done
|
|
(( verbose )) && echo ""
|
|
(( !syncing )) && break
|
|
if (( total_rest > 0 )); then
|
|
sleep 60
|
|
else
|
|
(( max_wait-- < 0 )) && break
|
|
sleep 1
|
|
fi
|
|
done
|
|
(( verbose )) && echo "$(date) sync appears to have finished at '$host_list'"
|
|
}
|
|
|
|
function migrate_resource
|
|
{
|
|
local source_primary="$1"
|
|
local target_primary="$2"
|
|
local target_secondary="$3"
|
|
local res="$4"
|
|
|
|
wait_resource_uptodate "$target_primary" "$res"
|
|
|
|
# critical path
|
|
section "Stopping old primary"
|
|
|
|
call_hook hook_resource_stop "$source_primary" "$res"
|
|
|
|
section "Migrate cluster config"
|
|
|
|
call_hook hook_migrate_cm3_config "$source_primary" "$target_primary" "$res"
|
|
|
|
section "Starting new primary"
|
|
|
|
call_hook hook_resource_start "$target_primary" "$res"
|
|
|
|
section "Checking new primary"
|
|
|
|
call_hook hook_resource_check "$res"
|
|
}
|
|
|
|
function migrate_cleanup
|
|
{
|
|
local host_list="$1"
|
|
local host_list2="$2"
|
|
local res="$3"
|
|
|
|
section "Cleanup migration data at $host_list"
|
|
|
|
local host
|
|
for host in $host_list; do
|
|
local vg_name="$(get_vg "$host")"
|
|
if [[ "$vg_name" != "" ]]; then
|
|
remote "$host" "marsadm wait-cluster || echo IGNORE cleanup"
|
|
remote "$host" "marsadm down $res || echo IGNORE cleanup"
|
|
remote "$host" "marsadm leave-resource $res || marsadm leave-resource --force $res || echo IGNORE cleanup"
|
|
remote "$host" "lvremove $lvremove_opt /dev/$vg_name/$res$tmp_suffix || echo IGNORE cleanup"
|
|
remote "$host" "lvremove $lvremove_opt /dev/$vg_name/$res-copy || echo IGNORE cleanup"
|
|
remote "$host" "lvremove $lvremove_opt /dev/$vg_name/$res$shrink_suffix_old || echo IGNORE cleanup"
|
|
remote "$host" "lvremove $lvremove_opt /dev/$vg_name/$res || echo IGNORE cleanup"
|
|
sleep 3
|
|
fi
|
|
done
|
|
|
|
section "Recompute host list"
|
|
|
|
local new_host_list="$(echo $(
|
|
for host in $host_list $host_list2; do
|
|
echo "$host"
|
|
remote "$host" "marsadm lowlevel-ls-host-ips" 2>/dev/null
|
|
done |\
|
|
awk '{ print $1; }' |\
|
|
sort -u ))"
|
|
echo "Augmented host list: $new_host_list"
|
|
host_list="$new_host_list"
|
|
|
|
for host in $host_list; do
|
|
remote "$host" "marsadm wait-cluster || echo IGNORE cleanup"
|
|
done
|
|
|
|
section "Split cluster at $host_list"
|
|
|
|
sleep 10
|
|
call_hook hook_prepare_hosts "$host_list"
|
|
call_hook hook_split_cluster "$host_list"
|
|
call_hook hook_finish_hosts "$host_list"
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# checks for FS shrinking
|
|
|
|
function determine_space
|
|
{
|
|
# works on global variables
|
|
lv_path="$(remote "$primary" "lvs --noheadings --separator ':' -o \"vg_name,lv_name\"" | grep ":$res$" | sed 's/ //g' | awk -F':' '{ printf("/dev/%s/%s", $1, $2); }')" || fail "cannot determine lv_path"
|
|
|
|
vg_name="$(echo "$lv_path" | cut -d/ -f3)" || fail "cannot determine vg_name"
|
|
|
|
echo "Determined the following VG name: \"$vg_name\""
|
|
echo "Determined the following LV path: \"$lv_path\""
|
|
|
|
local dev="/dev/$vg_name/$lv_name"
|
|
remote "$primary" "if [[ -e ${dev}$shrink_suffix_old ]]; then echo \"REFUSING to overwrite ${dev}$shrink_suffix_old on $primary - First remove it - Do this by hand\"; exit -1; fi"
|
|
|
|
df="$(remote "$hyper" "df $mnt" | grep "/dev/")" || fail "cannot determine df data"
|
|
used_space="$(echo "$df" | awk '{print $3;}')"
|
|
total_space="$(echo "$df" | awk '{print $2;}')"
|
|
# absolute or relative space computation
|
|
case "$target_percent" in
|
|
*k)
|
|
target_space="${target_percent%k}"
|
|
;;
|
|
*m)
|
|
target_space="$(( ${target_percent%m} * 1024 ))"
|
|
;;
|
|
*g)
|
|
target_space="$(( ${target_percent%g} * 1024 * 1024 ))"
|
|
;;
|
|
*)
|
|
target_space="${target_space:-$(( used_space * 100 / target_percent + 1 ))}" || fail "cannot compute target_space"
|
|
;;
|
|
esac
|
|
(( target_space < min_space )) && target_space=$min_space
|
|
|
|
echo "Determined USED space: $used_space"
|
|
echo "Determined TOTAL space: $total_space"
|
|
echo "Computed TARGET space: $target_space"
|
|
}
|
|
|
|
function check_shrinking
|
|
{
|
|
# works on global variables
|
|
if (( target_space >= total_space )); then
|
|
echo "No need for shrinking the LV space of $res"
|
|
(( !force )) && exit 0
|
|
fi
|
|
for host in $primary $secondary_list; do
|
|
check_vg_space "$host" "$target_space"
|
|
done
|
|
}
|
|
|
|
function check_extending
|
|
{
|
|
# works on global variables
|
|
if (( target_space <= total_space )); then
|
|
echo "No need for extending the LV space of $res"
|
|
(( !force )) && exit 0
|
|
fi
|
|
delta_space="$(( target_space - total_space + 1024 ))"
|
|
echo "Computed DELTA space: $delta_space"
|
|
for host in $primary $secondary_list; do
|
|
check_vg_space "$host" "$delta_space"
|
|
done
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# actions for FS shrinking
|
|
|
|
optimize_dentry_cache="${optimize_dentry_cache:-1}"
|
|
|
|
mkfs_cmd="${mkfs_cmd:-mkfs.xfs -s size=4096 -d agcount=1024}"
|
|
mount_opts="${mount_opts:--o rw,nosuid,noatime,attr2,inode64,usrquota}"
|
|
reuse_mount="${reuse_mount:-1}"
|
|
reuse_lv="${reuse_lv:-1}"
|
|
do_quota="${do_quota:-2}" # 1 = global xfs quota transfer, 2 = additionally local one
|
|
xfs_dump_dir="${xfs_dump_dir:-xfs-quota-$start_stamp}"
|
|
xfs_quota_enable="${xfs_quota_enable:-xfs_quota -x -c enable}"
|
|
xfs_dump="${xfs_dump:-xfs_quota -x -c dump}"
|
|
xfs_restore="${xfs_restore:-xfs_quota -x -c restore}"
|
|
|
|
function transfer_quota
|
|
{
|
|
local hyper="$1"
|
|
local lv_name="$2"
|
|
local mnt1="$3" # needs to be already mounted
|
|
local mnt2="$4" # needs to be already mounted
|
|
|
|
(( !do_quota )) && return
|
|
|
|
section "Checks for xfs quota transfer"
|
|
|
|
remote "$hyper" "mountpoint $mnt1 && mountpoint $mnt2"
|
|
|
|
section "Transfer xfs quota"
|
|
|
|
mkdir -p "$xfs_dump_dir"
|
|
local dumpfile="$xfs_dump_dir/xfs_dump.global.$hyper.$lv_name"
|
|
|
|
# enable quota
|
|
remote "$hyper" "$xfs_quota_enable $m2"
|
|
|
|
# transfer quota
|
|
remote "$hyper" "$xfs_dump $mnt1" > $dumpfile
|
|
ls -l $dumpfile
|
|
wc -l $dumpfile
|
|
if [[ -s $dumpfile ]]; then
|
|
local dev_name="$(remote "$hyper" "df $mnt2" | grep /dev/ | awk '{ print $1; }')"
|
|
echo "dev_name=$dev_name"
|
|
{
|
|
echo "fs = $dev_name"
|
|
tail -n +2 < $dumpfile
|
|
} > $dumpfile.new
|
|
remote "$hyper" "$xfs_restore $mnt2" < $dumpfile.new
|
|
else
|
|
echo "QUOTA IS EMPTY"
|
|
fi
|
|
}
|
|
|
|
function create_shrink_space
|
|
{
|
|
local host="$1"
|
|
local lv_name="$2"
|
|
local size="$3"
|
|
|
|
# some checks
|
|
section "Checking shrink space on $host"
|
|
|
|
local vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
remote "$host" "if [[ -e /dev/$vg_name/${lv_name}$shrink_suffix_old ]]; then echo \"REFUSING to overwrite /dev/$vg_name/${lv_name}$shrink_suffix_old on $host - Do this by hand\"; exit -1; fi"
|
|
if (( reuse_lv )); then
|
|
# check whether LV already exists
|
|
if remote "$host" "[[ -e /dev/$vg_name/${lv_name}$tmp_suffix ]]" 1; then
|
|
echo "reusing already exists LV /dev/$vg_name/${lv_name}$tmp_suffix on '$host'"
|
|
return
|
|
fi
|
|
fi
|
|
call_hook hook_disconnect "$host" "$lv_name"
|
|
remote "$host" "if [[ -e /dev/$vg_name/${lv_name}$tmp_suffix ]]; then lvremove $lvremove_opt /dev/$vg_name/${lv_name}$tmp_suffix; fi"
|
|
|
|
# do it
|
|
section "Creating shrink space on $host"
|
|
|
|
local extra="$(get_stripe_extra "$host" "$vg_name")"
|
|
remote "$host" "lvcreate -L ${size}k $extra -n ${lv_name}$tmp_suffix $vg_name"
|
|
remote "$host" "$mkfs_cmd /dev/$vg_name/${lv_name}$tmp_suffix"
|
|
}
|
|
|
|
function create_shrink_space_all
|
|
{
|
|
local host_list="$1"
|
|
local lv_name="$2"
|
|
local size="$3"
|
|
|
|
local host
|
|
for host in $host_list; do
|
|
create_shrink_space "$host" "$lv_name" "$size" "$count"
|
|
done
|
|
}
|
|
|
|
# convention: add a suffix -tmp to the device and mountpoint names each
|
|
|
|
function make_tmp_mount
|
|
{
|
|
local hyper="$1"
|
|
local store="$2"
|
|
local lv_name="$3"
|
|
local suffix="${4:-$tmp_suffix}"
|
|
|
|
local mnt="$(call_hook hook_get_mountpoint "$lv_name")"
|
|
if (( reuse_mount )); then
|
|
section "Checking mount $mnt$suffix at $hyper"
|
|
if remote "$hyper" "mountpoint $mnt$suffix" 1; then
|
|
echo "Reusing already existing mount $mnt$suffix on $hyper"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
section "Creating mount $mnt$suffix at $hyper"
|
|
|
|
local vg_name="$(get_vg "$store")" || fail "cannot determine VG for host '$store'"
|
|
local dev_tmp="/dev/$vg_name/$lv_name$suffix"
|
|
if [[ "$store" != "$hyper" ]]; then
|
|
# create remote devices instead
|
|
local old_dev="$dev_tmp"
|
|
dev_tmp="$(call_hook hook_connect "$store" "$hyper" "$lv_name$suffix" 2>&1 | tee /dev/stderr | grep "^NEW_DEV" | cut -d: -f2)"
|
|
echo "using tmp dev '$dev_tmp'"
|
|
[[ "$dev_tmp" = "" ]] && fail "cannot setup remote device between hosts '$store' => '$hyper'"
|
|
fi
|
|
remote "$hyper" "mkdir -p $mnt$suffix"
|
|
remote "$hyper" "mount $mount_opts $dev_tmp $mnt$suffix"
|
|
}
|
|
|
|
function make_tmp_umount
|
|
{
|
|
local hyper="$1"
|
|
local store="$2"
|
|
local lv_name="$3"
|
|
local suffix="${4:-$tmp_suffix}"
|
|
|
|
section "Removing temporary mount from $hyper"
|
|
|
|
remote "$hyper" "if mountpoint $mnt$suffix/; then sync; umount $mnt$suffix/ || umount -f $mnt$suffix/; fi"
|
|
|
|
if [[ "$store" != "$hyper" ]]; then
|
|
sleep 1
|
|
call_hook hook_disconnect "$store" "$lv_name$suffix"
|
|
fi
|
|
}
|
|
|
|
function copy_data
|
|
{
|
|
local hyper="$1"
|
|
local lv_name="$2"
|
|
local suffix="${3:-$tmp_suffix}"
|
|
local nice="${4:-$rsync_nice}"
|
|
local add_opt="${5:-$rsync_opt_prepare}"
|
|
local repeat_count="${6:-$rsync_repeat_prepare}"
|
|
|
|
local time_cmd="/usr/bin/time -f 'rss=%M elapsed=%e'"
|
|
|
|
section "COPY DATA via rsync"
|
|
|
|
local mnt="$(call_hook hook_get_mountpoint "$lv_name")"
|
|
|
|
remote "$hyper" "for i in {1..$repeat_count}; do echo round=\$i; $nice $time_cmd rsync $rsync_opt $add_opt $mnt/ $mnt$suffix/; rc=\$?; echo rc=\$rc; if (( !rc || rc == 24 )); then exit 0; fi; echo RESTARTING \$(date); done; echo FAIL; exit -1"
|
|
transfer_quota "$hyper" "$lv_name" "$mnt" "$mnt$suffix"
|
|
remote "$hyper" "sync"
|
|
}
|
|
|
|
function hot_phase
|
|
{
|
|
local hyper="$1"
|
|
local primary="$2"
|
|
local secondary_list="$3"
|
|
local lv_name="$4"
|
|
local suffix="${5:-$tmp_suffix}"
|
|
|
|
local mnt="$(call_hook hook_get_mountpoint "$lv_name")"
|
|
local vg_name="$(get_vg "$primary")" || fail "cannot determine VG for host '$host'"
|
|
local dev="/dev/$vg_name/$lv_name"
|
|
local dev_tmp="$dev$suffix"
|
|
local mars_dev="/dev/mars/$lv_name"
|
|
|
|
# some checks
|
|
section "Checking some preconditions"
|
|
|
|
remote "$primary" "if ! [[ -e $dev_tmp ]]; then echo \"Cannot start hot phase: $dev_tmp is missing. Run 'prepare' first!\"; exit -1; fi"
|
|
local host
|
|
for host in $primary $secondary_list; do
|
|
vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
remote "$host" "blkid /dev/$vg_name/$lv_name || true"
|
|
remote "$host" "blkid /dev/$vg_name/$lv_name$suffix || true"
|
|
done
|
|
|
|
# additional temporary mount
|
|
make_tmp_mount "$hyper" "$primary" "$lv_name" "$suffix"
|
|
|
|
section "Last online incremental rsync"
|
|
|
|
copy_data "$hyper" "$lv_name" "$suffix" "time" "$rsync_opt_prepare" "$rsync_repeat_prepare"
|
|
# repeat for better dentry caching
|
|
copy_data "$hyper" "$lv_name" "$suffix" "time" "$rsync_opt_prepare" "$rsync_repeat_prepare"
|
|
|
|
call_hook hook_save_local_quota "$hyper" "$lv_name"
|
|
|
|
# go offline
|
|
section "Go offline"
|
|
if (( optimize_dentry_cache )) && exists_hook hook_resource_stop_vm ; then
|
|
# retain mountpoints
|
|
call_hook hook_resource_stop_vm "$hyper" "$lv_name"
|
|
else
|
|
optimize_dentry_cache=0
|
|
# stop completely
|
|
call_hook hook_resource_stop "$primary" "$lv_name"
|
|
|
|
remote "$primary" "marsadm primary $lv_name"
|
|
if [[ "$primary" != "$hyper" ]]; then
|
|
# create remote devices instead
|
|
mars_dev="$(call_hook hook_connect "$primary" "$hyper" "$lv_name" 2>&1 | tee /dev/stderr | grep "^NEW_DEV" | cut -d: -f2)"
|
|
echo "using tmp mars dev '$mars_dev'"
|
|
[[ "$mars_dev" = "" ]] && fail "cannot setup remote mars device between hosts '$primary' => '$hyper'"
|
|
fi
|
|
remote "$hyper" "mount $mount_opts $mars_dev $mnt/"
|
|
fi
|
|
|
|
section "Final rsync"
|
|
|
|
copy_data "$hyper" "$lv_name" "$suffix" "time" "$rsync_opt_hot" "$rsync_repeat_hot"
|
|
|
|
make_tmp_umount "$hyper" "$primary" "$lv_name" "$suffix"
|
|
remote "$hyper" "rmdir $mnt$suffix || true"
|
|
if (( optimize_dentry_cache )); then
|
|
call_hook hook_resource_stop_rest "$hyper" "$primary" "$lv_name"
|
|
else
|
|
remote "$hyper" "sync; umount $mnt/"
|
|
if [[ "$primary" != "$hyper" ]]; then
|
|
# remove intermediate remote device
|
|
sleep 1
|
|
call_hook hook_disconnect "$primary" "$lv_name"
|
|
fi
|
|
fi
|
|
|
|
remote "$primary" "marsadm wait-umount $lv_name"
|
|
remote "$primary" "marsadm secondary $lv_name"
|
|
|
|
section "IMPORTANT: destroying the MARS resource"
|
|
echo "In case of failure, you can re-establish MARS resources by hand."
|
|
echo ""
|
|
|
|
for host in $secondary_list $primary; do
|
|
remote "$host" "marsadm wait-cluster || echo IGNORE"
|
|
remote "$host" "marsadm down $lv_name"
|
|
remote "$host" "marsadm leave-resource $lv_name || marsadm leave-resource --force $lv_name"
|
|
sleep 3
|
|
done
|
|
remote "$primary" "marsadm delete-resource $lv_name"
|
|
|
|
section "CRITICAL: Renaming LVs and re-creating the MARS resource"
|
|
echo "In case of failure, you need to CHECK the correct version by hand."
|
|
echo ""
|
|
|
|
for host in $primary $secondary_list; do
|
|
vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
remote "$host" "lvrename $vg_name $lv_name ${lv_name}$shrink_suffix_old"
|
|
remote "$host" "lvrename $vg_name $lv_name$suffix $lv_name"
|
|
done
|
|
remote "$primary" "marsadm create-resource --force $lv_name $dev"
|
|
remote "$primary" "marsadm primary $lv_name"
|
|
|
|
section "IMPORTANT: go online again"
|
|
echo "In case of failure, you can re-establish cm3 and MARS resources by hand."
|
|
echo ""
|
|
|
|
call_hook hook_resource_start "$primary" "$lv_name"
|
|
|
|
section "Re-create the MARS replicas"
|
|
|
|
for host in $secondary_list; do
|
|
vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
dev="/dev/$vg_name/${lv_name}"
|
|
if exists_hook hook_join_resource; then
|
|
call_hook hook_join_resource "$primary" "$host" "$lv_name" "$dev"
|
|
else
|
|
remote "$host" "marsadm join-resource $lv_name $dev"
|
|
fi
|
|
done
|
|
|
|
call_hook hook_restore_local_quota "$hyper" "$lv_name"
|
|
|
|
section "Checking new container"
|
|
|
|
call_hook hook_resource_check "$lv_name"
|
|
}
|
|
|
|
function cleanup_old_remains
|
|
{
|
|
local host_list="$1"
|
|
local lv_name="$2"
|
|
|
|
section "Cleanup any old LVs"
|
|
|
|
local host
|
|
for host in $host_list; do
|
|
local vg_name="$(get_vg "$host")"
|
|
if [[ "$vg_name" != "" ]]; then
|
|
make_tmp_umount "$host" "$host" "$lv_name" "$tmp_suffix"
|
|
section "Removing LVs from $host"
|
|
remote "$host" "lvremove $lvremove_opt /dev/$vg_name/${lv_name}$tmp_suffix || echo IGNORE LV removal"
|
|
remote "$host" "lvremove $lvremove_opt /dev/$vg_name/${lv_name}$shrink_suffix_old || echo IGNORE LV removal"
|
|
else
|
|
echo "ERROR: cannot determine VG for host $host" >> /dev/stderr
|
|
fi
|
|
done
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# actions for _online_ FS extension / resizing
|
|
|
|
fs_resize_cmd="${fs_resize_cmd:-xfs_growfs -d}"
|
|
|
|
function extend_fs
|
|
{
|
|
local hyper="$1"
|
|
local primary="$2"
|
|
local secondary_list="$3"
|
|
local lv_name="$4"
|
|
local size="$5"
|
|
|
|
local mnt="$(call_hook hook_get_mountpoint "$res")"
|
|
|
|
# extend the LV first
|
|
section "Extend the LV"
|
|
|
|
local host
|
|
for host in $primary $secondary_list; do
|
|
local vg_name="$(get_vg "$host")" || fail "cannot determine VG for host '$host'"
|
|
local dev="/dev/$vg_name/$lv_name"
|
|
remote "$host" "lvresize -L ${size}k $dev"
|
|
done
|
|
|
|
section "Extend the MARS resource"
|
|
|
|
remote "$primary" "marsadm resize $lv_name"
|
|
sleep 1
|
|
|
|
# propagate new size over intermediate iSCSI
|
|
if [[ "$hyper" != "$primary" ]]; then
|
|
section "propagate new size over iSCSI"
|
|
call_hook hook_extend_iscsi "$hyper"
|
|
sleep 3
|
|
fi
|
|
|
|
section "Resize the filesystem"
|
|
|
|
remote "$hyper" "$fs_resize_cmd $mnt"
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# internal actions (using global parameters)
|
|
|
|
### for migration
|
|
|
|
function migrate_prepare
|
|
{
|
|
call_hook hook_prepare_hosts "$primary $secondary_list $target_primary $target_secondary"
|
|
|
|
migration_prepare "$primary" "$res" "$target_primary" "$target_secondary"
|
|
|
|
call_hook hook_finish_hosts "$primary $secondary_list $target_primary $target_secondary"
|
|
}
|
|
|
|
function migrate_wait
|
|
{
|
|
wait_resource_uptodate "$target_primary $target_secondary" "$res"
|
|
}
|
|
|
|
function migrate_check
|
|
{
|
|
call_hook hook_check_migrate "$primary" "$target_primary" "$res"
|
|
}
|
|
|
|
function migrate_finish
|
|
{
|
|
migrate_resource "$primary" "$target_primary" "$target_secondary" "$res"
|
|
}
|
|
|
|
function manual_migrate_config
|
|
{
|
|
call_hook hook_migrate_cm3_config "$primary" "$target_primary" "$res"
|
|
}
|
|
|
|
function migrate_clean
|
|
{
|
|
migrate_cleanup "$to_clean_old" "$to_clean_new" "$res"
|
|
cleanup_old_remains "$to_clean_new" "$res"
|
|
}
|
|
|
|
### for shrinking
|
|
|
|
function shrink_prepare
|
|
{
|
|
create_shrink_space_all "$primary $secondary_list" "$res" "$target_space"
|
|
make_tmp_mount "$hyper" "$primary" "$res"
|
|
copy_data "$hyper" "$res" "$tmp_suffix" "$rsync_nice" "$rsync_opt_prepare" "$rsync_repeat_prepare"
|
|
call_hook hook_save_local_quota "$hyper" "$res"
|
|
if (( !reuse_mount )); then
|
|
make_tmp_umount "$hyper" "$primary" "$res"
|
|
fi
|
|
}
|
|
|
|
function shrink_finish
|
|
{
|
|
hot_phase "$hyper" "$primary" "$secondary_list" "$res"
|
|
}
|
|
|
|
function shrink_cleanup
|
|
{
|
|
cleanup_old_remains "$primary $secondary_list" "$res"
|
|
}
|
|
|
|
### for extending
|
|
|
|
function extend_stack
|
|
{
|
|
extend_fs "$hyper" "$primary" "$secondary_list" "$res" "$target_space"
|
|
}
|
|
|
|
### global actions
|
|
|
|
function lv_clean
|
|
{
|
|
LV_cleanup "$primary" "$res" 1
|
|
}
|
|
|
|
######################################################################
|
|
|
|
# MAIN: get and check parameters, determine hosts and resources, run actions
|
|
|
|
commands_installed "$commands_needed"
|
|
|
|
ssh-add -l || fail "You must use ssh-agent and ssh-add with the proper SSH identities"
|
|
|
|
scan_args "$@"
|
|
|
|
{
|
|
echo "$0 $@"
|
|
|
|
git describe --tags
|
|
|
|
source_hooks
|
|
|
|
# special (manual) operations
|
|
case "${operation//-/_}" in
|
|
manual_config_update)
|
|
call_hook hook_update_cm3_config "$host"
|
|
exit $?
|
|
;;
|
|
esac
|
|
|
|
# optional: allow syntax "resource:hypervisor:storage"
|
|
if [[ "$res" =~ : ]]; then
|
|
rest="${res#*:}"
|
|
res="${res%%:*}"
|
|
if [[ "$rest" =~ : ]]; then
|
|
storage_host[$res]="${rest#*:}"
|
|
rest="${rest%:*}"
|
|
fi
|
|
hypervisor_host[$res]="${rest%:*}"
|
|
fi
|
|
|
|
if [[ "$res" = "" ]]; then
|
|
helpme
|
|
fail "No resource name parameter given"
|
|
fi
|
|
|
|
hyper="$(get_hyper "$res")" || fail "No current hypervisor hostname can be determined"
|
|
|
|
echo "Determined the following CURRENT hypervisor: \"$hyper\""
|
|
|
|
if exists_hook hook_get_flavour; then
|
|
flavour="$(hook_get_flavour "$hyper" 2>/dev/null)"
|
|
echo "Determined the following hypervisor FLAVOUR: \"$flavour\""
|
|
fi
|
|
|
|
primary="$(get_store "$res")" || fail "No current primary hostname can be determined"
|
|
|
|
echo "Determined the following CURRENT primary: \"$primary\""
|
|
|
|
for host in $hyper $primary; do
|
|
ping -c 1 "$host" > /dev/null || fail "Host '$host' is not pingable"
|
|
done
|
|
|
|
remote "$primary" "mountpoint /mars"
|
|
remote "$primary" "[[ -d /mars/ips/ ]]"
|
|
remote "$primary" "marsadm view $res"
|
|
|
|
if (( $(remote "$primary" "marsadm view-is-primary $res") <= 0 )); then
|
|
fail "Resource '$res' on host '$primary' is not in PRIMARY role"
|
|
fi
|
|
|
|
mnt="$(call_hook hook_get_mountpoint "$res")"
|
|
if [[ "$mnt" != "" ]]; then
|
|
remote "$hyper" "mountpoint $mnt"
|
|
fi
|
|
|
|
secondary_list="$(remote "$primary" "marsadm view-resource-members $res" | { grep -v "^$primary$" || true; })" || fail "cannot determine secondary_list"
|
|
|
|
echo "Determined the following secondaries: '$secondary_list'"
|
|
|
|
for host in $secondary_list; do
|
|
ping -c 1 "$host" || fail "Host '$host' is not pingable"
|
|
remote "$host" "mountpoint /mars > /dev/null"
|
|
remote "$host" "[[ -d /mars/ips/ ]]"
|
|
# if [[ "$operation" =~ migrate ]] && ! [[ "$operation" =~ finish ]]; then
|
|
# local check
|
|
# for check in $target_primary $target_secondary; do
|
|
# if [[ "$check" = "$host" ]]; then
|
|
# fail "target '$check' is also a secondary - this cannot work"
|
|
# fi
|
|
# done
|
|
# fi
|
|
done
|
|
|
|
# check connections (only for migration)
|
|
if [[ "$operation" =~ migrate ]] && ! [[ "$operation" =~ cleanup|wait ]]; then
|
|
check_migration
|
|
fi
|
|
|
|
if [[ "$operation" = migrate_cleanup ]]; then
|
|
to_clean_old="$(hook_determine_old_replicas "$primary" "$res" 2>&1 | tee /dev/stderr | grep "^FOREIGN" | cut -d: -f2)"
|
|
to_clean_new="$(hook_determine_new_replicas "$primary" "$res" 2>&1 | tee /dev/stderr | grep "^FOREIGN" | cut -d: -f2)"
|
|
if [[ "$to_clean_old$to_clean_new" = "" ]]; then
|
|
echo "NOTHING TO DO"
|
|
exit 0
|
|
fi
|
|
echo "-------------"
|
|
echo "Temporary ${res}${tmp_suffix} partitions + LVs will be removed from:"
|
|
echo "$to_clean_new"
|
|
echo "Stray ${res}${shrink_suffix_old} backup partitions + LVs (old versions before shrinking) will be removed from:"
|
|
echo "$to_clean_old"
|
|
elif [[ "$operation" = lv_cleanup ]]; then
|
|
LV_cleanup "$primary" "$res" 0
|
|
fi
|
|
|
|
# determine sizes and available space (only for extending / shrinking)
|
|
if [[ "$operation" =~ shrink ]] && ! [[ "$operation" =~ cleanup ]]; then
|
|
determine_space
|
|
check_shrinking
|
|
elif [[ "$operation" =~ extend ]]; then
|
|
determine_space
|
|
check_extending
|
|
fi
|
|
|
|
# confirmation
|
|
|
|
if [[ "$target_primary" != "" ]]; then
|
|
echo "Using the following TARGET primary: \"$target_primary\""
|
|
echo "Using the following TARGET secondary: \"$target_secondary\""
|
|
fi
|
|
|
|
do_confirm
|
|
|
|
(( verbose < 1 )) && verbose=1
|
|
|
|
# main: start the internal actions
|
|
echo "START $(date)"
|
|
|
|
case "${operation//-/_}" in
|
|
migrate_prepare)
|
|
migrate_prepare
|
|
;;
|
|
migrate_wait)
|
|
migrate_wait
|
|
;;
|
|
migrate_finish)
|
|
migrate_check
|
|
migrate_finish
|
|
;;
|
|
migrate)
|
|
migrate_check
|
|
migrate_prepare
|
|
migrate_wait
|
|
migrate_finish
|
|
;;
|
|
migrate_cleanup)
|
|
migrate_clean
|
|
;;
|
|
|
|
manual_migrate_config)
|
|
migrate_check
|
|
manual_migrate_config
|
|
;;
|
|
|
|
|
|
shrink_prepare)
|
|
shrink_prepare
|
|
;;
|
|
shrink_finish)
|
|
shrink_finish
|
|
;;
|
|
shrink_cleanup)
|
|
shrink_cleanup
|
|
;;
|
|
shrink)
|
|
shrink_prepare
|
|
shrink_finish
|
|
shrink_cleanup
|
|
;;
|
|
|
|
extend)
|
|
extend_stack
|
|
;;
|
|
|
|
lv_cleanup)
|
|
lv_clean
|
|
;;
|
|
|
|
*)
|
|
helpme
|
|
echo "Unknown operation '$operation'"
|
|
exit -1
|
|
;;
|
|
esac
|
|
|
|
echo "DONE $(date)"
|
|
} 2>&1 | log "$logdir" "logs$args_info.$start_stamp.$LOGNAME.log"
|