diff --git a/test_suite/mars/default-resource.conf b/test_suite/mars/default-resource.conf index eeca7ccd..b0e6c219 100644 --- a/test_suite/mars/default-resource.conf +++ b/test_suite/mars/default-resource.conf @@ -135,3 +135,15 @@ resource_mars_dir_full_warn_pattern_list[0]='EMERGENCY: the space on '$global_ma ## flag to indicate whether the network connection should be cut while ## synching (e.g. after join-resource or invalidate) resource_cut_network_connection_while_sync=0 + +## resource indexed array containing percentages of the filling degrees of +## /mars beyond which the resources are switched to emergency mode +declare -g -A resource_emergency_percentage +resource_emergency_percentage=( +[lv-1-2]=20 +[lv-2-2]=40 +[lv-3-2]=60 +) + +## flag, whether only one resource should be put to emergency mode +resource_put_only_one_to_emergency=1 diff --git a/test_suite/mars/modules/datadev_full.sh b/test_suite/mars/modules/datadev_full.sh index 54bb91ba..3e6b5c28 100644 --- a/test_suite/mars/modules/datadev_full.sh +++ b/test_suite/mars/modules/datadev_full.sh @@ -61,7 +61,7 @@ function datadev_full_dd_on_device lib_vmsg " filling $dev on $host (bs=$bs, count=$count)" dd_out=($(lib_remote_idfile $host \ - "yes $(printf '%0.1024d' $control_nr) | dd of=$dev bs=$bs count=$count 2>&1")) + "yes $(printf '%0.1024d' $control_nr) | dd of=$dev oflag=direct bs=$bs count=$count 2>&1")) rc=$? if [ $should_fail -eq 1 ]; then if [ $rc -eq 0 ]; then diff --git a/test_suite/mars/modules/lib_err.sh b/test_suite/mars/modules/lib_err.sh index 5fc36273..eee853fd 100644 --- a/test_suite/mars/modules/lib_err.sh +++ b/test_suite/mars/modules/lib_err.sh @@ -138,15 +138,17 @@ function lib_err_wait_for_error_messages local number_errmsg_req=$4 maxwait=$5 local count waited=0 rc - lib_vmsg " checking existence of file $msg_file on $host" - lib_remote_idfile $host "ls -l --full-time $msg_file" || lib_exit 1 while true; do - count=$(lib_remote_idfile $host \ - "egrep '$errmsg_pattern' $msg_file | wc -l") || lib_exit 1 - lib_vmsg " found $count messages (pattern = '$errmsg_pattern'), waited $waited" - if [ $count -ge $number_errmsg_req ]; then - break + lib_vmsg " checking existence of file $msg_file on $host" + if lib_remote_idfile $host "ls -l --full-time $msg_file"; then + count=$(lib_remote_idfile $host \ + "egrep '$errmsg_pattern' $msg_file | wc -l") || lib_exit 1 + lib_vmsg " found $count messages (pattern = '$errmsg_pattern'), waited $waited" + if [ $count -ge $number_errmsg_req ]; then + break + fi fi + lib_vmsg " waited $waited for $msg_file to exist or $number_errmsg_req to be found in $msg_file" let waited+=1 sleep 1 if [ $waited -ge $maxwait ]; then diff --git a/test_suite/mars/modules/marsadm.sh b/test_suite/mars/modules/marsadm.sh index 99b0459c..b001ed15 100644 --- a/test_suite/mars/modules/marsadm.sh +++ b/test_suite/mars/modules/marsadm.sh @@ -1,4 +1,23 @@ -#!/bin/sh +#!/bin/bash +# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### function marsadm_do_cmd { @@ -308,3 +327,12 @@ function marsadm_primary_force marsadm_do_cmd $host "primary --force" "$res" || lib_exit 1 marsadm_do_cmd $host "connect" "$res" || lib_exit 1 } + +# without checks and debug messages +function marsadm_do_pur_cmd +{ + [ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 cmd="$2" cmd_args="$3" + lib_remote_idfile $host "marsadm $cmd --timeout=$marsadm_timeout $cmd_args" + return $? +} diff --git a/test_suite/mars/modules/mount.sh b/test_suite/mars/modules/mount.sh index c8667e21..ef089005 100644 --- a/test_suite/mars/modules/mount.sh +++ b/test_suite/mars/modules/mount.sh @@ -82,9 +82,8 @@ function mount_mount_data_device function mount_umount_data_device_all { - local res_no=${1:-0} + local res=${1:-${resource_name_list[0]}} local host - local res=${resource_name_list[$res_no]} for host in ${global_host_list[@]}; do mount_umount_data_device $host $res done diff --git a/test_suite/mars/modules/resource.sh b/test_suite/mars/modules/resource.sh index 3a364989..b650fbeb 100644 --- a/test_suite/mars/modules/resource.sh +++ b/test_suite/mars/modules/resource.sh @@ -23,6 +23,9 @@ function resource_prepare if [ $net_clear_iptables_in_prepare_phase -eq 1 ]; then net_clear_iptables_all fi + for res in ${resource_name_list[@]}; do + mount_umount_data_device_all $res + done resource_kill_all_scripts cluster_rmmod_mars_all cluster_clear_and_umount_mars_dir_all @@ -241,7 +244,6 @@ function resource_fill_mars_dir local mars_dev=$(lv_config_get_lv_device $mars_lv_name) local mars_dev_size=$(lv_config_get_lv_size_from_name $mars_lv_name) local time_waited writer_pid writer_script write_count control_nr - local primary_cksum secondary_cksum if [ $resource_use_data_dev_writes_to_fill_mars_dir -eq 1 ]; then resource_dd_until_mars_dir_full $primary_host $res \ @@ -273,9 +275,7 @@ function resource_fill_mars_dir lib_rw_start_writing_data_device $primary_host "writer_pid" \ "writer_script" 0 3 $res "" - local procfile=/proc/sys/mars/logger_resume - lib_vmsg " setting $primary_host:$procfile to 1" - lib_remote_idfile $primary_host "echo 1 >$procfile" || lib_exit 1 + resource_logger_resume $primary_host marsadm_do_cmd $secondary_host "invalidate" $res lib_wait_for_initial_end_of_sync $primary_host $secondary_host $res \ @@ -289,8 +289,15 @@ function resource_fill_mars_dir marsview_wait_for_state $secondary_host $res "disk" "Uptodate" \ $resource_maxtime_state_constant || lib_exit 1 - lib_rw_compare_checksums $primary_host $secondary_host $res 0 \ - "primary_cksum" "secondary_cksum" + lib_rw_compare_checksums $primary_host $secondary_host $res 0 "" "" +} + +function resource_logger_resume +{ + local host=$1 + local procfile=/proc/sys/mars/logger_resume + lib_vmsg " setting $host:$procfile to 1" + lib_remote_idfile $host "echo 1 >$procfile" || lib_exit 1 } function resource_resize_mars_dir @@ -342,7 +349,7 @@ function resource_check_low_space_error if [ -z "$msgpattern" ]; then lib_exit 1 "pattern resource_mars_dir_full_${msgtype}_pattern_list[$err_type] not found" fi - lib_err_wait_for_error_messages $host $msgfile "$msgpattern" 1 1 + lib_err_wait_for_error_messages $host $msgfile "$msgpattern" 1 10 } function resource_dd_until_mars_dir_full @@ -797,3 +804,124 @@ function resource_recreate_standalone lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited" lib_rw_compare_checksums $primary_host $secondary_host $res 0 "" "" } + +function resource_per_resource_emergency +{ + local primary_host=${global_host_list[0]} + local secondary_host=${global_host_list[1]} + local mars_lv_name=${cluster_mars_dir_lv_name_list[$primary_host]} + local mars_dev_size_mb=$((1024 * \ + $(lv_config_get_lv_size_from_name $mars_lv_name))) + + local dev=$(lv_config_get_lv_device $res) + local data_dev=$(resource_get_data_device $res) + local data_dev_size=$(lv_config_get_lv_size_from_name $res) + local mars_dev=$(lv_config_get_lv_device $mars_lv_name) + local time_waited writer_pid writer_script write_count control_nr + declare -A writer_script_per_resource + + for res in ${resource_name_list[@]}; do + mount_mount_data_device $primary_host $res + marsadm_do_cmd $primary_host "emergency-limit" "$res 0" || lib_exit 1 + lib_rw_start_writing_data_device $primary_host "writer_pid" \ + "writer_script" 0 4 $res $res + writer_script_per_resource[$res]=$writer_script + done + + if [ $resource_put_only_one_to_emergency -eq 1 ]; then + resource_test_emergency_on_one_resource $primary_host $secondary_host \ + $mars_lv_name $mars_dev_size_mb lv-1-2 + else + resource_test_emergency_on_all_resources $primary_host $secondary_host \ + $mars_lv_name $mars_dev_size_mb + fi + for res in ${resource_name_list[@]}; do + marsadm_do_cmd $primary_host "emergency-limit" "$res 0" || lib_exit 1 + done + +} + +# when entering this function it's assumed that a process is actually writing +# on the primary host +function resource_test_emergency_on_one_resource +{ + [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local primary_host=$1 secondary_host=$2 mars_lv_name=$3 mars_dev_size_mb=$4 + local res=$5 + local emergency_percentage=${resource_emergency_percentage[$res]} + local fill_size_mb time_waited + local marsadm_out host + + if [ -z "$emergency_percentage" ]; then + lib_exit 1 " missing value in resource_emergency_percentage for resource $res" + fi + + fill_size_mb=$(( ($mars_dev_size_mb * $emergency_percentage) / 100 )) + + lib_vmsg " creating $resource_big_file with $fill_size_mb MB to put $res in emerg. mode on $primary_host" + + datadev_full_dd_on_device $primary_host $resource_big_file $fill_size_mb \ + 4811 9 + + marsadm_do_cmd $primary_host "emergency-limit" \ + "$res $emergency_percentage" || lib_exit 1 + + resource_check_low_space_error $primary_host $res "sequence_hole" + + marsadm_out=$(marsadm_do_pur_cmd $primary_host "view-is-emergency" "$res") \ + || lib_exit 1 + if [ "$marsadm_out" != "1" ]; then + lib_vmsg " invalid output of marsadm view-is-emergency $res: $marsadm_out" >&2 + lib_vmsg " df -h $global_mars_directory on $primary_host:" >&2 + lib_remote_idfile $primary_host "df -h $global_mars_directory" >&2 + lib_exit 1 + fi + + resource_check_proc_sys_mars_emergency_file $primary_host + + lib_vmsg " removing $primary_host:$resource_big_file" + lib_remote_idfile $primary_host "rm -f $resource_big_file" || lib_exit 1 + + resource_logger_resume $primary_host + + marsadm_do_cmd $secondary_host "invalidate" $res + lib_wait_for_initial_end_of_sync $primary_host $secondary_host $res \ + $resource_maxtime_initial_sync \ + $resource_time_constant_initial_sync \ + "time_waited" + for host in $primary_host $secondary_host; do + resource_check_logfile_change $host $res + done + + lib_rw_compare_checksums $primary_host $secondary_host $res 0 "" "" + +} + +function resource_check_logfile_change +{ + local host=$1 res=$2 + local last_logfile last_logfile_old length length_old + local waited=0 maxwait=10 + + lib_vmsg " checking whether logfiles are written on $res on $host" + last_logfile_old=$(marsadm_get_last_logfile $host $res $host) || lib_exit 1 + length_old=$(file_handling_get_file_length \ + $host $last_logfile_old) || lib_exit 1 + lib_vmsg " start: last logfile:length on $host: $last_logfile_old:$length_old" + while true; do + last_logfile=$(marsadm_get_last_logfile $host $res $host) || lib_exit 1 + length=$(file_handling_get_file_length $host $last_logfile) || \ + lib_exit 1 + lib_vmsg " act.: last logfile:length on $host: $last_logfile:$length" + if [ "$last_logfile" != "$last_logfile_old" -o $length -ne $length_old ] + then + break + fi + sleep 1 + let waited+=1 + lib_vmsg " waited $waited for logfiles to change for $res on $host" + if [ $waited -eq $maxwait ]; then + lib_exit 1 "maxwait $maxwait exceeded" + fi + done +} diff --git a/test_suite/mars/test_cases/hardcore/all_resources.conf b/test_suite/mars/test_cases/hardcore/all_resources.conf new file mode 100644 index 00000000..edfe086f --- /dev/null +++ b/test_suite/mars/test_cases/hardcore/all_resources.conf @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +resource_put_only_one_to_emergency=0 diff --git a/test_suite/mars/test_cases/hardcore/mars_dir_full/per_resource_emergency/all_resources/i_am_a_testdirectory b/test_suite/mars/test_cases/hardcore/mars_dir_full/per_resource_emergency/all_resources/i_am_a_testdirectory new file mode 100644 index 00000000..e69de29b diff --git a/test_suite/mars/test_cases/hardcore/mars_dir_full/per_resource_emergency/only_one_resource/i_am_a_testdirectory b/test_suite/mars/test_cases/hardcore/mars_dir_full/per_resource_emergency/only_one_resource/i_am_a_testdirectory new file mode 100644 index 00000000..e69de29b diff --git a/test_suite/mars/test_cases/hardcore/only_one_resource.conf b/test_suite/mars/test_cases/hardcore/only_one_resource.conf new file mode 100644 index 00000000..fe62053b --- /dev/null +++ b/test_suite/mars/test_cases/hardcore/only_one_resource.conf @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +resource_put_only_one_to_emergency=1 diff --git a/test_suite/mars/test_cases/hardcore/per_resource_emergency.conf b/test_suite/mars/test_cases/hardcore/per_resource_emergency.conf new file mode 100644 index 00000000..d7300c3a --- /dev/null +++ b/test_suite/mars/test_cases/hardcore/per_resource_emergency.conf @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + + +resource_fs_on_data_device_necessary=1 + +lib_rw_part_of_device_size_written_per_loop=$((1024 * 1024)) + +## maxtime to wait for secondary to become disk state = Uptodate and +## repl state = Replaying +mars_dir_full_maxtime_state_constant=60 + + +cluster_mars_dir_lv_name_list=([${global_host_list[0]}]=lv-5-10 [${global_host_list[1]}]=lv-6-100) + +resource_name_list=( +lv-1-2 +lv-2-2 +lv-3-2 +) + +resource_set_globals_depending_on_resource_name_list + +run_list="resource_prepare resource_run_all resource_per_resource_emergency lib_general_mars_checks_after_every_test"