mars/contrib/example-scripts/run-masses.sh

797 lines
29 KiB
Bash
Raw Normal View History

#!/bin/bash
#
# This file is part of MARS project: http://schoebel.github.io/mars/
#
# Copyright (C) 2015 Thomas Schoebel-Theuer
# Copyright (C) 2015 1&1 Internet AG
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
############################################################
# TST autumn 2015 lab prototype
# for mass switchover and other generic mass commands
# Environment-specific actions are encoded into variables.
# Change them (e.g. in /etc/mass-actions/mass-actions.conf) for
# adaptation to any other operating environment.
#
# In addition, you will need an association file host-assoc.txt
# containing 2 fields separated by colon: first the hostname, second
# an arbitrary key value with an arbitrary meaning. It is wise to
# use locations, room numbers, rack numbers, etc for that field.
# What exactly is up to you. Multiple keys may be assigned to the same
# host.
#
# Please feel free to adapt this to your needs.
set -o pipefail
shopt -s nullglob
export LC_ALL=C
export start_stamp="$(date "+%F_%T" | sed 's/:/./g')"
declare -A doc
orig_vars="$(set | grep '^[_A-Za-z0-9]\+=' | cut -d= -f1)"
# START defaults for configuration variables
default_config="${default_config:-./mass-actions.conf}"
doc[default_config]="Default config file. Here you can override variables or add additional commands to the array cmd_table[]."
additional_configs="${additional_configs:-/etc/mass-actions/*.conf $HOME/.mass-actions/*.conf}"
doc[additional_configs]="Blank-separated list of wildcarded additional config files. Here you can override variables or add additional commands to the array cmd_table[]."
status_dir="${status_dir:-./status-dir}"
doc[status_dir]="Output directory where progress logfiles of remotely issued ssh commands are created. You may grep in it."
# The rest is hardcoded here in case the config file does not exist
dry_run=${dry_run:-0}
doc[dry_run]="When enabled, remote ssh actions are only displayed instead of really executed."
verbose=${verbose:-0}
doc[verbose]="Increase speakyness."
confirm=${confirm:-0}
doc[confirm]="Each remote ssh command must be individually confirmed before it is actually executed. As a side effect, commands are running sequentially instead of parallel."
do_wait=${do_wait:-1}
allow_unknown_hosts=${allow_unknown_hosts:-0}
help=${help:-0}
status=${status:-0}
clean=${clean:-0}
sshopt="${sshopt:--4 -A -T -o StrictHostKeyChecking=no -o ForwardX11=no -o KbdInteractiveAuthentication=no -o VerifyHostKeyDNS=no -o ConnectTimeout=60 -o TCPKeepAlive=yes}"
max_jobs_parallelism="${max_jobs_parallelism:-3000}"
host_spec="${host_spec:-}"
action="${action:-}"
cmd="${cmd:-}"
prefix_cmd="${prefix_cmd:-set -o pipefail; shopt -s nullglob; }"
host_list="${host_list:-}"
host_filter="${host_filter:-}"
skip=${skip:-0}
max=${max:-0}
assoc_file="${assoc_file:-host-assoc.txt}"
assoc_dirs="${assoc_dirs:-. $HOME/.mass-actions /var/cache/mass-actions /etc/mass-actions}"
txt_ok="${txt_ok:-CMD OK}"
txt_fail="${txt_fail:-CMD FAIL \$?}"
# Command table for defining shorthand actions.
# Replace or extend this for your needs.
#
# Hint: use /etc/mass-actions/mass-actions.conf (or put it at another place)
# for overriding these example commands.
#
# All available action keywords can displayed by "$0 --help".
# Variables starting with tmp_ are suppressed in the display and may
# be used for internal structuring / better readability of complex commands.
declare -A cmd_table
cmd_table[test]="uptime"
cmd_table[mars_status]="if [[ -d /mars ]]; then marsadm view-replstate all; else echo 'NO_MARS_HOST'; fi"
cmd_table[drbd_status]="if [[ -r /proc/drbd ]]; then cat /proc/drbd; else echo 'NO_DRBD_HOST'; fi"
cmd_table[cm3_status]="cm3 -us || cm3 -s"
cmd_table[kernel_status]="uptime; ${cmd_table[cm3_status]}; ${cmd_table[mars_status]}; ${cmd_table[drbd_status]}; available=\"\$(ls -t /boot/vmlinuz-* | head -1 | cut -d- -f2-)\"; echo AVAILABLE_KERNEL=\$available; running=\"\$(cat /proc/version | awk '{print \$3; }')\"; echo RUNNING_KERNEL=\$running; if [[ \"\$running\" = \"\$available\" ]]; then echo KERNEL_IS_RECENT; elif [[ -r /proc/drbd ]] && grep ':Primary/' < /proc/drbd; then echo CANNOT_REBOOT_DRBD_PRIMARY; elif [[ -d /mars ]] && marsadm view-is-primary all | grep '^1\$'; then echo CANNOT_REBOOT_MARS_PRIMARY; elif [[ -x /usr/lib/1und1/scripts/is_node_in_mode_active.sh ]] && /usr/lib/1und1/scripts/is_node_in_mode_active.sh; then echo CANNOT_REBOOT_NODE_ACTIVE; else echo NEEDS_REBOOT; fi"
cmd_table[cm3_switchable_status]="if [[ -d /etc/ovz ]]; then cm3_switchable=1; else cm3_switchable=0; for dummy in {0..3}; do cm3 -us; slots_needed=\"\$(cm3 -s | grep \" \(remote\\|stopped\|broken\) \" | wc -l)\"; slots_available=\"\$(cm3 -s | grep idle | wc -l)\"; if (( slots_needed <= slots_available )); then cm3_switchable=1; break; fi; sleep 7; echo CM3_REPEAT; done; if (( cm3_switchable )); then echo CM3_SWITCHOVER_POSSIBLE; else echo CM3_SWITCHOVER_NOT_POSSIBLE; fi; fi"
cmd_table[mars_module_status]="uptime; mars_available=\"\$(modinfo mars | grep '^version' | awk '{ print \$2; }')\"; echo \"AVAILABLE_MARS=\$mars_available\"; mars_running=\"\$(cat /sys/module/mars/version | awk '{ print \$1; }')\"; echo \"RUNNING_MARS=\$mars_running\"; if [[ \"\$mars_running\" = \"\" ]]; then echo echo 'NO_MARS_HOST'; elif [[ \"\$mars_running\" = \"\$mars_available\" ]]; then echo MARS_IS_RECENT; elif marsadm view-is-primary all | grep '^1\$'; then echo MARS_CANNOT_RELOAD; else echo MARS_NEEDS_RELOAD; fi"
cmd_table[bgp_status]="if mountpoint /kunden/homepages/; then if ping -c 1 -w 10 8.8.8.8; then echo BGP_OK; else echo BGP_FAIL; fi; else echo BGP_UNUSED; fi"
cmd_table[detect_double]="if [[ -r /proc/drbd ]]; then cat /proc/drbd; if grep ' ds:' < /proc/drbd && mountpoint /mars && [[ -h /mars/uuid ]]; then marsadm view all; echo DOUBLE; else echo 'NO_MARS_HOST'; fi; else echo 'NO_DRBD_HOST'; fi"
cmd_table[kernel_reboot_when_necessary]="if { ${cmd_table[kernel_status]}; } | tee -a /dev/stderr | grep -q '^NEEDS_REBOOT$'; then if [[ -r /etc/lilo.conf ]] && grep rtrfix < /etc/lilo.conf; then lilo && sleep 3 && lilo -R rtrfix && sleep 3 && sync && echo coldreboot && coldreboot; else echo reboot; reboot; fi; fi"
cmd_table[mars_reload_when_necessary]="if { ${cmd_table[mars_module_status]}; } | tee -a /dev/stderr | grep -q '^MARS_NEEDS_RELOAD$'; then rmmod mars; modprobe mars; fi"
cmd_table[mars_switchover]="if [[ -d /mars ]]; then marsadm up all; marsadm primary all; fi; ${cmd_table[mars_status]}"
cmd_table[mars_failover]="if [[ -d /mars ]]; then marsadm pause-fetch all; marsadm attach all; marsadm primary --force all; fi; ${cmd_table[mars_status]}"
cmd_table[drbd_switchover]="if [[ -r /proc/drbd ]]; then drbdadm up all; drbdadm primary all; fi; ${cmd_table[drbd_status]}"
cmd_table[drbd_failover]="if [[ -r /proc/drbd ]]; then drbdadm disconnect all; drbdadm primary --force all; fi; ${cmd_table[drbd_status]}"
tmp_cm3_options="--timeout=3600 --vmhandler-timeout=3600"
tmp_mars_detect_others="export resources=\"\$(marsadm view-my-resources)\"; other_hosts=\"\"; for res in \$resources; do primary=\"\$(marsadm view-get-primary \$res)\"; if [[ \"\$primary\" != \"\$(hostname)\" ]] && ! [[ \"\$other_hosts\" =~ \$primary ]]; then other_hosts+=\" \$primary\"; fi; done"
tmp_mars_check_switchable="if ! [[ -d /proc/sys/mars ]]; then echo 'CANNOT_START_MARS_SWITCHOVER: kernel module not loaded'; exit -1; fi; if marsadm view-is-attach all | grep -q \"^0\$\"; then echo 'CANNOT_START_MARS_SWITCHOVER: some resource not attached'; exit -1; fi; if marsadm view-is-alive all | grep -v \"^---\" | grep -v \"^1\$\"; then echo 'CANNOT_START_MARS_SWITCHOVER: network is not alive'; exit -1; fi; if marsadm view-sync-rest all | grep -v \"^---\" | grep -v \"^0\$\"; then echo 'CANNOT_START_MARS_SWITCHOVER: some resource not synced'; exit -1; fi; if marsadm view-is-split-brain all | grep -v \"^---\" | grep -v \"^0\$\"; then echo 'CANNOT_START_MARS_SWITCHOVER: some resource is in split brain'; exit -1; fi; if marsadm view-is-consistent all | grep -v \"^---\" | grep -v \"^1\$\"; then echo 'CANNOT_START_MARS_SWITCHOVER: some resource is inconsistent'; exit -1; fi"
tmp_drbd_detect_others="export resources=\"\$(if [[ -d /etc/ovz/drbd.conf.d/ ]]; then (cd /etc/ovz/drbd.conf.d/ && echo \$(ls ovz*.cfg ovz*.cfg.old | cut -d. -f1 | sort -u) ); else echo \$(cm3 --list-vms | cut -d. -f1); fi)\"; if grep -q ':Secondary/' < /proc/drbd; then other_hosts=\"\$(hostname | tr ab ba)\"; fi"
tmp_drbd_check_switchable="if grep \" cs:\" < /proc/drbd | grep -v \"cs:Connected .* ds:UpToDate/UpToDate\"; then echo 'CANNOT_START_DRBD_SWITCHOVER'; exit -1; fi"
tmp_cm3_stop_other="ssh $sshopt root@\$host \"$prefix_cmd cm3 $tmp_cm3_options --stop all; sleep 20; count=0; for i in \\\$(cm3 --list-vms --with-status | grep -i broken | cut -d: -f1 | cut -d. -f1); do echo \"RESTOPPING BROKEN \\\$i\"; (( count++ )); sleep 20; cm3 -us; sleep 10; cm3 $tmp_cm3_options --stop \\\$i; done\""
tmp_rebuild_ovz_tmp="for dev in /dev/vg*/ovz[0-9]*tmp; do if grep \"\$(echo \$dev | sed 's:^.*/::')\" < /proc/mounts; then echo \"Cannot rebuild \$dev\"; else echo \"Rebuild \$dev\"; if mkfs.xfs -f \$dev; then mount \$dev /mnt; chmod a+rwxt /mnt; umount /mnt; fi; fi; done"
#tmp_cm3_restart_local="for dummy in {0..2\}; do count=0; for i in \$(cm3 --list-vms --with-status | grep -i \"broken\|stopped\" | cut -d: -f1 | cut -d. -f1); do echo \"RESTARTING BROKEN \$i\"; (( count++ )); cm3 -us; sleep 10; cm3 $tmp_cm3_options --stop \$i; done; if (( count )); then sleep 10; cm3 $tmp_cm3_options --start all; sleep 10; fi; done"
tmp_cm3_restart_local="echo skip restart"
tmp_cm3_start_local="$tmp_rebuild_ovz_tmp; cm3 $tmp_cm3_options --start all; sleep 10; cm3 -us; $tmp_cm3_restart_local"
tmp_cm3_status_local="${cmd_table[mars_status]}; ${cmd_table[drbd_status]}; cm3 -us; cm3 -s | grep -q 'broken\|stopped' && exit -1"
tmp_mars_restart_cmd="drbdadm down all; /etc/init.d/drbd stop; sleep 3; /etc/init.d/drbd stop; sleep 3; rmmod drbd; sleep 1; modprobe mars"
# Problem: ssh evaluates its arguments once more. Solution: for symmetry reasons, use eval at the local side to get the same number of evaluations. Use enough backslashes to distinguish between the different numbers of evaluation levels.
tmp_mars_update_configs_resources_cmd="if which configure_InfongSpace.pl; then configure_InfongSpace.pl --update-infong \\\$res repltype=mars; elif which ui-config-modify; then ui-config-modify -c MARS_ENABLED=true; fi"
tmp_mars_make_resources_primary="echo RESOURCES \$resources; for res in \$resources; do echo marsadm create-resource \$res /dev/*/\$res; marsadm create-resource \\\$res /dev/*/\\\$res || exit -1; $tmp_mars_update_configs_resources_cmd; done"
tmp_mars_make_resources_secondary="echo RESOURCES \$resources; for res in \$resources; do echo marsadm join-resource \\\$res /dev/*/\\\$res; marsadm join-resource \\\$res /dev/*/\\\$res || exit -1; $tmp_mars_update_configs_resources_cmd; done"
tmp_update_configs_cmd="for i in /etc/ovz/drbd.conf.d/*.cfg; do mv \\\$i \\\$i.MARS; done; if [[ -r /etc/ovz/fstab.include ]]; then for file in /etc/ovz/fstab.include /etc/fstab; do sed --in-place=.MARS 's:\(/dev/drbd[0-9]\+\) \+/vz/\([0-9]\+\):/dev/mars/ovz\\2 /vz/\\2:' \\\$file; done; fi"
cmd_table[fix_mars_config]="eval \"$tmp_update_configs_cmd\""
tmp_restart_cm3_cmd="/etc/init.d/clustermanager stop; sleep 3; marsadm secondary all; /etc/init.d/clustermanager start; sleep 20"
tmp_mars_make_resources="if [[ -h /mars/uuid ]]; then $tmp_mars_restart_cmd; ssh $sshopt root@\$other_hosts \"$prefix_cmd $tmp_mars_restart_cmd\"; eval \"$tmp_mars_make_resources_primary\"; eval \"$tmp_update_configs_cmd\"; sleep 10; res=SCHEISSE; ssh $sshopt root@\$other_hosts \"$prefix_cmd $tmp_mars_make_resources_secondary; $tmp_update_configs_cmd\"; $tmp_restart_cm3_cmd; ssh $sshopt root@\$other_hosts \"$prefix_cmd $tmp_restart_cm3_cmd\"; fi"
tmp_mars_create_cluster="if ! [[ -h /mars/uuid ]]; then ssh $sshopt root@\$other_hosts \"mount /mars; marsadm create-cluster\"; marsadm join-cluster \$other_hosts; fi"
tmp_mars_migrate="mount /mars; if [[ \"\$other_hosts\" != \"\" ]] && [[ \"\$resources\" != \"\" ]] && [[ -r /proc/drbd ]] && grep ' ro:' < /proc/drbd && mountpoint /mars && ! grep 'ro:Primary/' < /proc/drbd && ! grep -o -i 'ds:[a-z/]\+' < /proc/drbd | grep -v 'UpToDate/UpToDate'; then echo \"---- MIGRATING \$(hostname) (\$other_hosts) [\$resources] ------\"; $tmp_mars_create_cluster; $tmp_mars_make_resources; fi"
#tmp_mars_migrate="echo WEGLASSEN"
cmd_table[cm3_get_resources]="if [[ -d /sys/module/mars/ ]] ; then $tmp_mars_check_switchable; $tmp_mars_detect_others; elif [[ -r /proc/drbd ]]; then $tmp_drbd_check_switchable; $tmp_drbd_detect_others; else echo 'NO_CM3_RUNNING'; exit 0; fi; for res in \$resources; do echo \"\$res:\$(ls /dev/*/\$res | grep -v /mars | tail -1)\"; done"
cmd_table[cm3_switchover]="${cmd_table[cm3_get_resources]}; if [[ \"\$resources\" = \"\" ]]; then echo NO_RESOURCES_EXIST; exit 0; fi; echo \"other_hosts='\$other_hosts'\"; ${cmd_table[cm3_switchable_status]}; if (( !cm3_switchable )); then exit -1; fi; for host in \$other_hosts; do echo \"---- STOPPING \$host ------\"; $tmp_cm3_stop_other; sleep 10; done; $tmp_mars_migrate; echo \"---- STARTING \$(hostname) ------\"; sleep 10; $tmp_cm3_start_local; sleep 10; $tmp_cm3_status_local; ${cmd_table[bgp_status]}; exit 0"
cmd_table[repair_ovz_drbd]="/etc/init.d/drbd stop; /etc/init.d/clustermanager stop; /etc/init.d/drbd stop; rmmod mars; umount /mars; for i in /etc/ovz/drbd.conf.d/*.cfg.MARS /etc/ovz/fstab.include.MARS /etc/fstab.MARS; do mv \$i \${i/.MARS/}; done; /etc/init.d/drbd start; /etc/init.d/clustermanager start; mkfs.ext4 /dev/vg00/mars; mount /mars"
# The following functions may be overridden in the config file.
# When new functions are declared, their function names must follow
# the convention print_[a-z0-9_]+_status()
#
# Any new functions are automatically detected and included.
#
# Typically, they will grep in the output of previously defined remote commands
# and display some statistics about the contents.
#
# Important: these functions should not print anything when no data
# is available.
function print_ping_status
{
local output="$(cat $status_dir/*.log |\
grep -o " packets transmitted, [0-9]\+ received" |\
awk '{ print $3; }' |\
sort -n |\
uniq -c |\
awk '{ printf(" %s=%d", $2, $1); }')"
if [[ "$output" != "" ]]; then
echo " PING STATUS:$output"
fi
}
function print_ssh_status
{
local msg_list="Host.key.verification.failed Permission.denied Connection.refused Connection.timed.out Could.not.resolve.hostname unknown.host"
local output="$(cat $status_dir/*.log |\
grep -o "\(${msg_list// /\\|}\)" |\
sed 's/ /_/g' |\
sort |\
uniq -c |\
awk '{ printf(" %s=%d", $2, $1); }')"
if [[ "$output" != "" ]]; then
echo " SSH STATUS:$output"
fi
}
function print_uptime_status
{
local day_limits="0 1 7 30 365"
local load_limits="0 1 3 10 30 100 300"
local count=0
local limit
for limit in $day_limits; do
eval local days_$limit=$(
cat $status_dir/*.log |\
grep -o "up [0-9]\+ days," |\
awk "{ if (\$2 >= $limit) { print \$2} }" |\
wc -l)
(( days_$limit && count++ ))
done
for limit in $load_limits; do
eval local load_$limit=$(
cat $status_dir/*.log |\
grep -o "load average: [0-9]\+" |\
awk "{ if (\$3 >= $limit) { print \$3} }" |\
wc -l)
(( load_$limit && count++ ))
done
if (( count )); then
echo -n " UPTIME:"
for limit in $day_limits; do
echo -n " >${limit}_days=$(eval echo "\${days_$limit}")"
done
echo ""
echo -n " LOADAVG:"
for limit in $load_limits; do
echo -n " >${limit}=$(eval echo "\${load_$limit}")"
done
echo ""
fi
}
function print_kernel_status
{
local msg_list="KERNEL_IS_RECENT CANNOT_REBOOT[A-Z_]* NEEDS_REBOOT"
local output="$(cat $status_dir/*.log |\
grep -o "^\(${msg_list// /\\|}\)$" |\
sort |\
uniq -c |\
awk '{ printf(" %s=%d", $2, $1); }')"
if [[ "$output" != "" ]]; then
echo " KERNEL STATUS:$output"
fi
}
function print_mars_status
{
local msg_list="NO_MARS_HOST MARS_IS_RECENT MARS_CANNOT_RELOAD[A-Z_]* MARS_NEEDS_RELOAD"
local output="$(cat $status_dir/*.log |\
grep -o "^\(${msg_list// /\\|}\)$" |\
sort |\
uniq -c |\
awk '{ printf(" %s=%d", $2, $1); }')"
if [[ "$output" != "" ]]; then
echo " MARS STATUS:$output"
fi
local msg_list="ModuleNotLoaded UnResponsive NotJoined NotStarted EmergencyMode Replicating NotYetPrimary PausedSync Syncing PausedFetch PausedReplay NoPrimaryDesignated PrimaryUnreachable Replaying"
local msg
for msg in $msg_list; do
eval "local $msg=0"
done
local count=0
local var
for var in $(cat $status_dir/*.log | grep -o "^\(${msg_list// /\\|}\)$"); do
(( count++ ))
eval "(( $var++ ))"
done
if (( count )); then
echo -n " MARS RESOURCES:"
for msg in $msg_list; do
if (( $(eval echo \${$msg}) )); then
echo -n " $msg=$(eval echo \${$msg})"
fi
done
echo ""
fi
}
function print_drbd_status
{
local output="$(cat $status_dir/*.log |\
grep -i -o 'NO_DRBD_HOST\| cs:[a-z]\+\| ro:[a-z/]\+\| ds:[a-z/]\+' |\
sed 's/^ *[a-z]\+://' |\
sort |\
uniq -c |\
awk '{ printf(" %s=%d", $2, $1); }')"
if [[ "$output" != "" ]]; then
echo " DRBD RESOURCES:$output"
fi
}
function print_cm3_status
{
local msg_list="NO_CM3_RUNNING NO_RESOURCES_EXIST CANNOT_START_DRBD_SWITCHOVER CANNOT_START_MARS_SWITCHOVER CM3_SWITCHOVER_POSSIBLE CM3_SWITCHOVER_NOT_POSSIBLE"
local found=0
local var;
for var in $msg_list; do
eval "local ${var//./_}=0";
done
local var
for var in $(cat $status_dir/*.log | grep -o "^\(${msg_list// /\\|}\)" | sed 's/ /_/g'); do
(( found++ ))
eval "(( ${var//./_}++ ))"
done
if (( found )); then
echo -n " CM3 STATUS:"
local msg
for msg in $msg_list; do
if (( $(eval echo \${${msg//\./_}}) )); then
echo -n " ${msg//\./_}=$(eval echo \${${msg//\./_}})"
fi
done
echo ""
fi
local key_list="started stopped active remote broken disabled"
found=0
local key
for key in $key_list; do
eval "local nr_$key=0"
done
for file in $status_dir/*.log; do
# determine the last line, in case there are multiple invocations
# of "cm3 -s" in the same logfile.
line="$(grep -n "VM *.*STATE *NODE *STORAGE" < $file | tail -1 | cut -d: -f1)"
if [[ "$line" != "" ]]; then
(( found++ ))
for key in $(tail -n +$line < $file | grep -o " \(${key_list// /\\|}\) .*" | awk '{ print $1; rest=$2; while (rest = gensub("[^,]*,?", "", "", rest)) { print $1; } }'); do
eval "(( nr_$key++ ))"
done
fi
done
if (( found )); then
echo -n " CM3 RESOURCES:"
for key in $key_list; do
echo -n " $key=$(eval echo \${nr_$key})"
done
echo ""
fi
}
function print_bgp_status
{
local output="$(cat $status_dir/*.log |\
grep '^\(BGP_[A-Z_]\+\)$' |\
sort -r |\
uniq -c |\
awk '{ printf(" %s=%d", $2, $1); }')"
if [[ "$output" != "" ]]; then
echo " BGP STATUS:$output"
fi
}
# END of configuration variables and functions
param_vars="$(set | grep '^[_A-Za-z0-9]\+=' | cut -d= -f1 | grep -v "^tmp_")"
########################################################
# generic helper functions
function warn
{
local txt="${1:-Unkown}"
echo "WARNING: $txt" >> /dev/stderr
}
function fail
{
local txt="${1:-Unkown failure}"
echo "FAILURE: $txt" >> /dev/stderr
rm -f /tmp/tmp_*.$$
exit -1
}
function do_confirm
{
local skip_this="$1"
local active="${2:-$confirm}"
local response
(( !active )) && return 0
[[ "$skip_this" != "" ]] && skip_this="S to skip, "
echo -n "[CONFIRM: Press ${skip_this}Return to continue, ^C to abort] "
read response
! [[ "$response" =~ ^[sS] ]]
return $?
}
function remote
{
local host="$1"
local cmd="$2"
local nofail="${3:-0}"
(( verbose > 1 )) && echo "Executing on $host: '$cmd'" >> /dev/stderr
[[ "${cmd## }" = "" ]] && return 0
if ssh $sshopt root@$host "$cmd"; then
return 0
elif (( nofail )); then
return $?
else
#fail "ssh to '$host' command '$cmd' failed with status $?"
fail "ssh to '$host' command failed with status $?"
fi
}
function remote_action
{
local host="$1"
local cmd="$2"
if (( dry_run )); then
echo "DRY_RUN REMOTE $host ACTION '$cmd'"
elif (( confirm )); then
echo "REMOTE $host ACTION '$cmd'"
if do_confirm 1; then
remote "$host" "$cmd"
else
echo "SKIPPING $host ACTION '$cmd'"
fi
else
remote "$host" "$cmd"
fi
}
function source_when_possible
{
local file="$1"
local type="$2"
if [[ -r "$file" ]]; then
echo "Sourcing $type file '$file'"
. "$file" || fail "$type file $file is not parsable"
elif (( verbose )); then
echo "Skipping non-existent $type file '$file'"
fi
}
for i; do
if [[ "$i" =~ ^--verbose ]]; then
verbose=1
fi
done
for file in $additional_configs; do
source_when_possible "$file" "config"
done
source_when_possible "$default_config" "config"
# Allow forceful override of any _known_ variable at the command line
for i; do
if [[ "$i" =~ ^--[-_A-Za-z0-9]+$ ]]; then
param="${i#--}"
var="${param//-/_}"
[[ "$(eval "echo \"\${$var-UNSET}\"")" = "UNSET" ]] && fail "Variable '$var' is unknown"
eval "$var=1"
elif [[ "$i" =~ ^--[-_A-Za-z0-9]+= ]]; then
param="${i#--}"
var="${param%%=*}"
var="${var//-/_}"
val="${param#*=}"
[[ "$(eval "echo \"\${$var-UNSET}\"")" = "UNSET" ]] && fail "Variable '$var' is unknown"
eval "$var=$val"
elif [[ "$i" =~ ^-h$ ]]; then
help=1
elif [[ "$i" =~ ^-v$ ]]; then
(( verbose++ ))
elif [[ "$host_spec" = "" ]]; then
host_spec="$i"
elif [[ "$action" = "" ]]; then
action="$i"
else
fail "bad parameter syntax '$i'"
fi
done
for dir in $assoc_dirs; do
if [[ -r "$dir/$assoc_file" ]]; then
assoc_file="$dir/$assoc_file"
break
fi
done
function do_help
{
cat <<EOF
usage: $0 [options] <host_spec> <action>
---------------------
The following parameter variables can be either passed by the
environment, or used for hard overriding on the command line
via --variable=value syntax:
$(
declare -A orig
for i in $orig_vars; do
orig[$i]=1
done
for i in $param_vars; do
[[ "$i" =~ _vars$ ]] && continue
if (( !orig[$i] )); then
if [[ "$(eval "echo \${$i}")" =~ ^[0-9]+$ ]]; then
echo "$i=$(eval "echo \${$i}")"
else
echo "$i=\"$(eval "echo \${$i}")\""
fi
doc_line="${doc[$i]}"
if [[ "$doc_line" != "" ]]; then
echo -e "\t$doc_line"
fi
fi
done
)
---------------------
The following status functions are defined and are automatically called
upon $0 --status :
$(set | grep "^[a-z0-9_]\+ ()" | grep "^print_[a-z0-9_]\+_status")
---------------------
The following strings can be used for <host_spec>:
(see file $assoc_file)
$(cut -d: -f2 < $assoc_file | sort -u)
Hint: multiple specs may be separated by blanks, if you correctly
quote it to the shell. Example: $0 "host1 host7" "uptime"
Set operations can be performed by prefixing each spec or hostname with
"+" or "-" signs.
Example: $0 "+de.kae.bs -de.kae.bs;R08" "kernel_status"
will run on all hosts from complete datacenter "de.kae.bs" with the
exception of all hosts from Room 08.
Filtering: $0 --host-filter="store" "de.kae.bs" "kernel_status"
will only run on final target hostnames containing the substring "store".
You may also use bash regexes.
---------------------
The following pre-defined <action>s from cmd_table[] can be used (or,
give a full shell command in quotes):
$(
local i
for i in ${!cmd_table[*]}; do
echo "$i"
done
)
EOF
}
if (( help )); then
do_help
exit 0
fi
function print_status
{
local empty=0
local failure=0
local ok=0
local working=0
local file
for file in $status_dir/*.log; do
if ! [[ -s $file ]]; then
(( empty++ ))
elif grep -q FAILURE $file; then
(( failure++ ))
elif grep -q "^$txt_ok$" $file; then
(( ok++ ))
else
(( working++ ))
fi
done
echo "REMOTE SCRIPT STATUS: NotStarted=$empty Working=$working OK=$ok Fail=$failure"
local func
for func in $(set | grep "^[a-z0-9_]\+ ()" | grep -o "^print_[a-z0-9_]\+_status"); do
$func
done
}
if (( status )); then
[[ -d "$status_dir" ]] || fail "Status directory '$status_dir' does not exist"
sub_dir="$(ls $status_dir | grep "^run-" | sort | tail -1)"
[[ -d "$status_dir/$sub_dir" ]] && export status_dir="$status_dir/$sub_dir"
echo "Status from $status_dir:"
print_status
exit 0
fi
if (( clean )); then
[[ -d "$status_dir" ]] || fail "Status directory '$status_dir' does not exist"
echo "Are you sure to clean the status directory $status_dir/ including all its versioned subdirectories?"
do_confirm 1 1
rm -rf $status_dir
exit 0
fi
# automatic versioning of status_dir
export status_dir="$status_dir/run-$start_stamp"
########################################################
# compute host_list out of host_spec
function add_host
{
local host="$1"
local minus="$2"
if (( minus )); then
host_list="$(echo " $host_list " | sed "s/ $host / /g")"
else
host_list+=" $host"
fi
}
function compute_host_list
{
local host
rm -f /tmp/tmp_*.$$
local tmp1=/tmp/tmp_1.$$
local tmp2=/tmp/tmp_2.$$
[[ -r $assoc_file ]] || fail "cannot find assoc file '$assoc_file'"
(( verbose )) && echo "Using assoc file '$assoc_file'"
for host in $host_spec; do
local minus=0
if [[ "$host" =~ ^- ]]; then
host="${host/-/}"
minus=1
else
host="${host/\+/}"
fi
host="${host//./\\.}"
if grep -E ":$host\$" < $assoc_file > $tmp1; then
local i
for i in $(cut -d: -f1 < $tmp1); do
add_host $i $minus
done
elif grep -qE "^$host:" < $assoc_file; then
add_host $host $minus
elif (( allow_unknown_hosts )); then
warn "host '$host' does not appear in $assoc_file"
add_host $host $minus
else
fail "Keyword or hostname '$host' does not exist in $assoc_file"
fi
done
rm -f /tmp/tmp_*.$$
if [[ "$host_filter" != "" ]]; then
local old_host_list="$host_list"
host_list=""
for host in $old_host_list; do
if [[ "$host" =~ $host_filter ]]; then
host_list+=" $host"
fi
done
fi
if (( skip > 0 )); then
local old_host_list="$host_list"
local count=0
host_list=""
for host in $old_host_list; do
(( ++count <= skip )) && continue
if [[ "$host" =~ $host_filter ]]; then
host_list+=" $host"
fi
done
fi
if (( max > 0 )); then
local old_host_list="$host_list"
local count=0
host_list=""
for host in $old_host_list; do
(( ++count > max )) && break
if [[ "$host" =~ $host_filter ]]; then
host_list+=" $host"
fi
done
fi
local host_count=$(echo ${host_list} | wc -w)
if (( !host_count )); then
fail "Resulting host list is empty - nothing can be done at all"
fi
if (( verbose )); then
echo "USING FINAL host_list: ${host_list}"
else
echo "Will run on $host_count hosts"
fi
}
function get_cmd
{
if [[ "$cmd" = "" ]]; then
if [[ "$action" = "ping" ]]; then
echo "Running a pure ping to $(echo "$host_list" | wc -w) hosts"
cmd="ping"
elif [[ "${cmd_table[$action]}" != "" ]]; then
echo "Using predefined cmd_table[] action '$action'"
if ! [[ "$action" =~ _status ]]; then
do_confirm 1 1
fi
cmd="$prefix_cmd${cmd_table[$action]}"
elif [[ "$action" != "" ]]; then
echo ""
echo "Running action '$action' as a command on $(echo "$host_list" | wc -w) hosts"
do_confirm 1 1
cmd="$action"
else
fail "No action given."
fi
else
echo ""
echo "Using given command '$cmd' on $(echo "$host_list" | wc -w) hosts"
do_confirm 1 1
fi
}
########################################################
# main program
function main
{
mkdir -p $status_dir || fail "connot create status directory '$status_dir'"
script_start=$(date +%s)
if (( confirm )); then
echo "CONFIRM mode: everything is running SEQUENTIALLY"
else
echo "START forking sub-processes"
fi
local host
for host in $host_list; do
if (( confirm )); then
if remote_action $host "$cmd" 2>&1; then
eval echo "$txt_ok"
else
eval echo "$txt_fail"
fi 2>&1 | tee $status_dir/$host.log
else
if (( dry_run )); then
echo "DRY_RUN REMOTE $host ACTION '$cmd'"
eval echo "$txt_ok"
elif [[ "$cmd" = "ping" ]]; then
ping -c 1 -w 10 $host
eval echo "$txt_ok"
elif remote $host "$cmd" 2>&1 ; then
eval echo "$txt_ok"
else
eval echo "$txt_fail"
fi > $status_dir/$host.log 2>&1 &
while (( $(jobs | wc -l) > max_jobs_parallelism )); do
sleep 1
done
fi
done
(( !confirm )) && echo "DONE forking sub-processes"
if (( do_wait )); then
echo "Waiting for termination of sub-processes"
local duration=1
while (( $( pstree $$ | wc -l ) > 2 )); do
print_status
sleep $duration
(( duration < 10 && duration++ ))
done
wait
fi
script_end=$(date +%s)
echo "ESTIMATED script duration: $(( script_end - script_start )) seconds"
print_status
}
compute_host_list
get_cmd
main
exit 0