RepoMirrors
/
mars
mirror of https://github.com/schoebel/mars
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test_suite: new test cases WIP

This commit is contained in:
Frank Liepold 2014-03-11 08:07:32 +01:00 committed by Thomas Schoebel-Theuer
parent 55e80074d9
commit 56b6b382a7
11 changed files with 273 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
test_suite/mars/default-resource.conf
View File

@ -135,3 +135,15 @@ resource_mars_dir_full_warn_pattern_list[0]='EMERGENCY: the space on '$global_ma
## flag to indicate whether the network connection should be cut while
## synching (e.g. after join-resource or invalidate)
resource_cut_network_connection_while_sync=0
## resource indexed array containing percentages of the filling degrees of
## /mars beyond which the resources are switched to emergency mode
declare -g -A resource_emergency_percentage
resource_emergency_percentage=(
[lv-1-2]=20
[lv-2-2]=40
[lv-3-2]=60
)
## flag, whether only one resource should be put to emergency mode
resource_put_only_one_to_emergency=1

2
test_suite/mars/modules/datadev_full.sh
View File

@ -61,7 +61,7 @@ function datadev_full_dd_on_device
lib_vmsg " filling $dev on $host (bs=$bs, count=$count)"
dd_out=($(lib_remote_idfile $host \
"yes $(printf '%0.1024d' $control_nr) | dd of=$dev bs=$bs count=$count 2>&1"))
"yes $(printf '%0.1024d' $control_nr) | dd of=$dev oflag=direct bs=$bs count=$count 2>&1"))
rc=$?
if [ $should_fail -eq 1 ]; then
if [ $rc -eq 0 ]; then

16
test_suite/mars/modules/lib_err.sh
View File

@ -138,15 +138,17 @@ function lib_err_wait_for_error_messages
local number_errmsg_req=$4 maxwait=$5
local count waited=0 rc
lib_vmsg " checking existence of file $msg_file on $host"
lib_remote_idfile $host "ls -l --full-time $msg_file" || lib_exit 1
while true; do
count=$(lib_remote_idfile $host \
"egrep '$errmsg_pattern' $msg_file | wc -l") || lib_exit 1
lib_vmsg " found $count messages (pattern = '$errmsg_pattern'), waited $waited"
if [ $count -ge $number_errmsg_req ]; then
break
lib_vmsg " checking existence of file $msg_file on $host"
if lib_remote_idfile $host "ls -l --full-time $msg_file"; then
count=$(lib_remote_idfile $host \
"egrep '$errmsg_pattern' $msg_file | wc -l") || lib_exit 1
lib_vmsg " found $count messages (pattern = '$errmsg_pattern'), waited $waited"
if [ $count -ge $number_errmsg_req ]; then
break
fi
fi
lib_vmsg " waited $waited for $msg_file to exist or $number_errmsg_req to be found in $msg_file"
let waited+=1
sleep 1
if [ $waited -ge $maxwait ]; then

30
test_suite/mars/modules/marsadm.sh
View File

@ -1,4 +1,23 @@
#!/bin/sh
#!/bin/bash
# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
function marsadm_do_cmd
{
@ -308,3 +327,12 @@ function marsadm_primary_force
marsadm_do_cmd $host "primary --force" "$res" || lib_exit 1
marsadm_do_cmd $host "connect" "$res" || lib_exit 1
}
# without checks and debug messages
function marsadm_do_pur_cmd
{
[ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local host=$1 cmd="$2" cmd_args="$3"
lib_remote_idfile $host "marsadm $cmd --timeout=$marsadm_timeout $cmd_args"
return $?
}

3
test_suite/mars/modules/mount.sh
View File

@ -82,9 +82,8 @@ function mount_mount_data_device
function mount_umount_data_device_all
{
local res_no=${1:-0}
local res=${1:-${resource_name_list[0]}}
local host
local res=${resource_name_list[$res_no]}
for host in ${global_host_list[@]}; do
mount_umount_data_device $host $res
done

142
test_suite/mars/modules/resource.sh
View File

@ -23,6 +23,9 @@ function resource_prepare
if [ $net_clear_iptables_in_prepare_phase -eq 1 ]; then
net_clear_iptables_all
fi
for res in ${resource_name_list[@]}; do
mount_umount_data_device_all $res
done
resource_kill_all_scripts
cluster_rmmod_mars_all
cluster_clear_and_umount_mars_dir_all
@ -241,7 +244,6 @@ function resource_fill_mars_dir
local mars_dev=$(lv_config_get_lv_device $mars_lv_name)
local mars_dev_size=$(lv_config_get_lv_size_from_name $mars_lv_name)
local time_waited writer_pid writer_script write_count control_nr
local primary_cksum secondary_cksum
if [ $resource_use_data_dev_writes_to_fill_mars_dir -eq 1 ]; then
resource_dd_until_mars_dir_full $primary_host $res \
@ -273,9 +275,7 @@ function resource_fill_mars_dir
lib_rw_start_writing_data_device $primary_host "writer_pid" \
"writer_script" 0 3 $res ""
local procfile=/proc/sys/mars/logger_resume
lib_vmsg " setting $primary_host:$procfile to 1"
lib_remote_idfile $primary_host "echo 1 >$procfile" || lib_exit 1
resource_logger_resume $primary_host
marsadm_do_cmd $secondary_host "invalidate" $res
lib_wait_for_initial_end_of_sync $primary_host $secondary_host $res \
@ -289,8 +289,15 @@ function resource_fill_mars_dir
marsview_wait_for_state $secondary_host $res "disk" "Uptodate" \
$resource_maxtime_state_constant || lib_exit 1
lib_rw_compare_checksums $primary_host $secondary_host $res 0 \
"primary_cksum" "secondary_cksum"
lib_rw_compare_checksums $primary_host $secondary_host $res 0 "" ""
}
function resource_logger_resume
{
local host=$1
local procfile=/proc/sys/mars/logger_resume
lib_vmsg " setting $host:$procfile to 1"
lib_remote_idfile $host "echo 1 >$procfile" || lib_exit 1
}
function resource_resize_mars_dir
@ -342,7 +349,7 @@ function resource_check_low_space_error
if [ -z "$msgpattern" ]; then
lib_exit 1 "pattern resource_mars_dir_full_${msgtype}_pattern_list[$err_type] not found"
fi
lib_err_wait_for_error_messages $host $msgfile "$msgpattern" 1 1
lib_err_wait_for_error_messages $host $msgfile "$msgpattern" 1 10
}
function resource_dd_until_mars_dir_full
@ -797,3 +804,124 @@ function resource_recreate_standalone
lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited"
lib_rw_compare_checksums $primary_host $secondary_host $res 0 "" ""
}
function resource_per_resource_emergency
{
local primary_host=${global_host_list[0]}
local secondary_host=${global_host_list[1]}
local mars_lv_name=${cluster_mars_dir_lv_name_list[$primary_host]}
local mars_dev_size_mb=$((1024 * \
$(lv_config_get_lv_size_from_name $mars_lv_name)))
local dev=$(lv_config_get_lv_device $res)
local data_dev=$(resource_get_data_device $res)
local data_dev_size=$(lv_config_get_lv_size_from_name $res)
local mars_dev=$(lv_config_get_lv_device $mars_lv_name)
local time_waited writer_pid writer_script write_count control_nr
declare -A writer_script_per_resource
for res in ${resource_name_list[@]}; do
mount_mount_data_device $primary_host $res
marsadm_do_cmd $primary_host "emergency-limit" "$res 0" || lib_exit 1
lib_rw_start_writing_data_device $primary_host "writer_pid" \
"writer_script" 0 4 $res $res
writer_script_per_resource[$res]=$writer_script
done
if [ $resource_put_only_one_to_emergency -eq 1 ]; then
resource_test_emergency_on_one_resource $primary_host $secondary_host \
$mars_lv_name $mars_dev_size_mb lv-1-2
else
resource_test_emergency_on_all_resources $primary_host $secondary_host \
$mars_lv_name $mars_dev_size_mb
fi
for res in ${resource_name_list[@]}; do
marsadm_do_cmd $primary_host "emergency-limit" "$res 0" || lib_exit 1
done
}
# when entering this function it's assumed that a process is actually writing
# on the primary host
function resource_test_emergency_on_one_resource
{
[ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local primary_host=$1 secondary_host=$2 mars_lv_name=$3 mars_dev_size_mb=$4
local res=$5
local emergency_percentage=${resource_emergency_percentage[$res]}
local fill_size_mb time_waited
local marsadm_out host
if [ -z "$emergency_percentage" ]; then
lib_exit 1 " missing value in resource_emergency_percentage for resource $res"
fi
fill_size_mb=$(( ($mars_dev_size_mb * $emergency_percentage) / 100 ))
lib_vmsg " creating $resource_big_file with $fill_size_mb MB to put $res in emerg. mode on $primary_host"
datadev_full_dd_on_device $primary_host $resource_big_file $fill_size_mb \
4811 9
marsadm_do_cmd $primary_host "emergency-limit" \
"$res $emergency_percentage" || lib_exit 1
resource_check_low_space_error $primary_host $res "sequence_hole"
marsadm_out=$(marsadm_do_pur_cmd $primary_host "view-is-emergency" "$res") \
|| lib_exit 1
if [ "$marsadm_out" != "1" ]; then
lib_vmsg " invalid output of marsadm view-is-emergency $res: $marsadm_out" >&2
lib_vmsg " df -h $global_mars_directory on $primary_host:" >&2
lib_remote_idfile $primary_host "df -h $global_mars_directory" >&2
lib_exit 1
fi
resource_check_proc_sys_mars_emergency_file $primary_host
lib_vmsg " removing $primary_host:$resource_big_file"
lib_remote_idfile $primary_host "rm -f $resource_big_file" || lib_exit 1
resource_logger_resume $primary_host
marsadm_do_cmd $secondary_host "invalidate" $res
lib_wait_for_initial_end_of_sync $primary_host $secondary_host $res \
$resource_maxtime_initial_sync \
$resource_time_constant_initial_sync \
"time_waited"
for host in $primary_host $secondary_host; do
resource_check_logfile_change $host $res
done
lib_rw_compare_checksums $primary_host $secondary_host $res 0 "" ""
}
function resource_check_logfile_change
{
local host=$1 res=$2
local last_logfile last_logfile_old length length_old
local waited=0 maxwait=10
lib_vmsg " checking whether logfiles are written on $res on $host"
last_logfile_old=$(marsadm_get_last_logfile $host $res $host) || lib_exit 1
length_old=$(file_handling_get_file_length \
$host $last_logfile_old) || lib_exit 1
lib_vmsg " start: last logfile:length on $host: $last_logfile_old:$length_old"
while true; do
last_logfile=$(marsadm_get_last_logfile $host $res $host) || lib_exit 1
length=$(file_handling_get_file_length $host $last_logfile) || \
lib_exit 1
lib_vmsg " act.: last logfile:length on $host: $last_logfile:$length"
if [ "$last_logfile" != "$last_logfile_old" -o $length -ne $length_old ]
then
break
fi
sleep 1
let waited+=1
lib_vmsg " waited $waited for logfiles to change for $res on $host"
if [ $waited -eq $maxwait ]; then
lib_exit 1 "maxwait $maxwait exceeded"
fi
done
}

22
test_suite/mars/test_cases/hardcore/all_resources.conf Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
resource_put_only_one_to_emergency=0

0
test_suite/mars/test_cases/hardcore/mars_dir_full/per_resource_emergency/all_resources/i_am_a_testdirectory Normal file
View File

0
test_suite/mars/test_cases/hardcore/mars_dir_full/per_resource_emergency/only_one_resource/i_am_a_testdirectory Normal file
View File

22
test_suite/mars/test_cases/hardcore/only_one_resource.conf Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
resource_put_only_one_to_emergency=1

42
test_suite/mars/test_cases/hardcore/per_resource_emergency.conf Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
resource_fs_on_data_device_necessary=1
lib_rw_part_of_device_size_written_per_loop=$((1024 * 1024))
## maxtime to wait for secondary to become disk state = Uptodate and
## repl state = Replaying
mars_dir_full_maxtime_state_constant=60
cluster_mars_dir_lv_name_list=([${global_host_list[0]}]=lv-5-10 [${global_host_list[1]}]=lv-6-100)
resource_name_list=(
lv-1-2
lv-2-2
lv-3-2
)
resource_set_globals_depending_on_resource_name_list
run_list="resource_prepare resource_run_all resource_per_resource_emergency lib_general_mars_checks_after_every_test"