RepoMirrors
/
mars
mirror of https://github.com/schoebel/mars
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test_suite: new test case for logging messages in syslogds logfile

This commit is contained in:
Frank Liepold 2014-03-13 14:49:00 +01:00 committed by Thomas Schoebel-Theuer
parent e21d048709
commit c831e34cec
6 changed files with 245 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
test_suite/mars/default-syslog.conf Normal file
View File

@ -0,0 +1,54 @@
#!/bin/bash
# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
## defaults for module syslog
## file where the mars messages are logged
syslog_logfile=/var/log/syslog
## prefix of files containing parameters for logging
syslog_paramfile_prefix=/proc/sys/mars/syslog_flood_
## file containing the message class beyond which messages are to be logged
syslog_flood_class_file=${syslog_paramfile_prefix}class
## file containing the number of messages which are logged within
## the recovery window (for further information see the mars manual)
syslog_flood_limit_file=${syslog_paramfile_prefix}limit
## file containing the duration of the recovery window
syslog_flood_recovery_s_file=${syslog_paramfile_prefix}recovery_s
## msg class beyond (incl.) which messages are logged
syslog_flood_class=3 # errors
## number of messages (see above)
syslog_flood_limit=20
## recovery time (see above)
syslog_flood_recovery_s=30
## error message pattern to egrep for
syslog_err_msg_pattern='.*(s_trans_logger.*replay error|make_log_finalize.*replay stopped with error)'
## time to wait for the syslog_msg_number_limit messages to appear
syslog_msg_wait_time=10

1
test_suite/mars/mars_test_cronjob.sh
View File

@ -184,6 +184,7 @@ tests_to_execute=(
"test_cases/admin/resizing:test_cases/admin"
"test_cases/admin/logrotate:test_cases/admin"
"test_cases/admin/logdelete:test_cases/admin"
"test_cases/admin/syslog_messages:test_cases/admin"
"test_cases/bugs/memleak:test_cases/bugs"
"test_cases/admin/leave_resource_while_sync*:test_cases/admin"
"test_cases/admin/switch2primary:test_cases/admin"

10
test_suite/mars/modules/lib_wait.sh
View File

@ -21,7 +21,7 @@ function lib_wait_until_logfile_has_length
{
[ $# -eq 7 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local host=$1 logfile=$2 length_logfile=$3
local varname_time_waited=$4 maxwait=$5 check_net_throughput=$6 varname_net_throughput=$7
local varname_time_waited=$4 maxwait=$5 check_net_throughput=$6 varname_net_throughput="$7"
local act_length
local waited=0 start_time=$(date +'%s') end_time
local my_net_throughput=0 net_throughput_sum=0 net_check_count=0
@ -68,7 +68,7 @@ function lib_wait_until_fetch_stops
[ $# -eq 9 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local module=$1 secondary_host=$2 primary_host=$3 res=$4
local varname_logfile=$5 varname_length_logfile=$6
local varname_time_waited=$7 check_net_throughput=$8 varname_net_throughput=$9
local varname_time_waited=$7 check_net_throughput=$8 varname_net_throughput="$9"
local maxtime_fetch time_constant_fetch var v
for var in maxtime_fetch time_constant_fetch; do
@ -85,7 +85,7 @@ function lib_wait_until_fetch_stops
$time_constant_fetch \
$varname_logfile $varname_length_logfile \
$varname_time_waited \
$check_net_throughput $varname_net_throughput
$check_net_throughput "$varname_net_throughput"
}
function lib_wait_internal_until_fetch_stops
@ -93,7 +93,7 @@ function lib_wait_internal_until_fetch_stops
[ $# -eq 10 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local secondary_host=$1 res=$2 primary_host=$3 maxwait=$4 inactive_wait=$5
local varname_logfile=$6 varname_logfile_length=$7 varname_time_waited=$8
local check_net_throughput=$9 varname_net_throughput=${10}
local check_net_throughput=$9 varname_net_throughput="${10}"
local inactive_waited=0 msg
local my_logfile length file_and_length file_and_length_old="x"
local waited=0 msg start_time=$(date +'%s') end_time
@ -156,7 +156,7 @@ function lib_wait_until_action_stops
[ $# -eq 8 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local action=$1 host=$2 res=$3 maxwait=$4 inactive_wait=$5
local varname_time_waited=$6 check_net_throughput=$7
local varname_net_throughput=$8
local varname_net_throughput="$8"
local waited=0 link_value link_value_old="x"
local inactive_waited=0 msg start_time=$(date +'%s') end_time
local link=$(lib_linktree_get_res_host_linkname $host $res $action)

160
test_suite/mars/modules/syslog.sh Normal file
View File

@ -0,0 +1,160 @@
#!/bin/bash
# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
function syslog_run
{
local primary_host=${global_host_list[0]}
local secondary_host=${global_host_list[1]}
local res=${resource_name_list[0]}
local writer_pid writer_script write_count
local logfile length_logfile time_waited
local logfile_sav=$global_mars_directory/logfile.sav.$$
local nr_msg_orig i
lib_wait_for_initial_end_of_sync $primary_host $secondary_host $res \
$resource_maxtime_initial_sync \
$resource_time_constant_initial_sync \
"time_waited"
lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited"
mount_mount_data_device $primary_host $res
# 2 loops to test recovery window
for i in 1 2; do
resource_clear_data_device $primary_host $res
lib_rw_start_writing_data_device $primary_host "writer_pid" \
"writer_script" 2 2 $res ""
marsadm_pause_cmd "apply" $secondary_host $res
lib_wait_until_action_stops "replay" $secondary_host $res \
$apply_fetch_maxtime_apply \
$apply_fetch_time_constant_apply \
"time_waited" 0 ""
lib_vmsg " ${FUNCNAME[0]}: apply time: $time_waited"
marsview_wait_for_state $secondary_host $res "disk" "Outdated\[.*A.*\]" \
$marsview_wait_for_state_time
marsview_wait_for_state $secondary_host $res "repl" '-SF--' \
$marsview_wait_for_state_time || lib_exit 1
lib_rw_stop_writing_data_device $primary_host $writer_script \
"write_count"
main_error_recovery_functions["lib_rw_stop_scripts"]=
lib_wait_until_fetch_stops "apply_fetch" $secondary_host $primary_host \
$res "logfile" "length_logfile" \
"time_waited" 0 ""
lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited"
file_handling_check_equality_of_file_lengths $logfile $primary_host \
$secondary_host \
$length_logfile
syslog_set_logging_parameters $secondary_host
lib_vmsg " copying $secondary_host:$logfile to $logfile_sav"
lib_remote_idfile $secondary_host \
"rm -f $logfile_sav && cp $logfile $logfile_sav" || \
lib_exit 1
file_destroy_dd_on_logfile $secondary_host $logfile $length_logfile
nr_msg_orig=$(syslog_count_or_check_messages $secondary_host \
$syslog_flood_limit 1) || lib_exit 1
marsadm_do_cmd $secondary_host "resume-replay" $res || lib_exit 1
syslog_count_or_check_messages $secondary_host \
$(( $nr_msg_orig + $syslog_flood_limit )) 0
# stopp generation of new error messages
marsadm_pause_cmd "apply" $secondary_host $res
lib_wait_until_action_stops "replay" $secondary_host $res \
$apply_fetch_maxtime_apply \
$apply_fetch_time_constant_apply \
"time_waited" 0 ""
lib_vmsg " restoring $logfile from $logfile_sav"
lib_remote_idfile $secondary_host \
"dd if=$logfile_sav of=$logfile conv=notrunc" || \
lib_exit 1
marsadm_do_cmd $secondary_host "resume-replay" $res || lib_exit 1
nr_msg_orig=$(syslog_count_or_check_messages $secondary_host \
$syslog_flood_limit 1) || lib_exit 1
lib_vmsg " sleeping syslog_recovery_s = $syslog_recovery_s seconds"
syslog_count_or_check_messages $secondary_host $nr_msg_orig 0
done
}
function syslog_set_logging_parameters
{
local host=$1 param file value varname filename
for param in class limit recovery_s; do
varname='syslog_flood_'$param
lib_vmsg " setting $varname on $host"
eval value='$'$varname
if [ -z "$value" ]; then
lib_exit 1 "no value found for variable $varname"
fi
filename='syslog_flood_'$param'_file'
eval file='$'$filename
lib_remote_idfile $host "ls -l $file" || lib_exit 1
lib_remote_idfile $host "echo $value > $file" || lib_exit 1
done
}
# we cannot use lib_err_wait_for_error_messages, because we need a grep -v egrep
# because sometimes all commands are logged in /var/log/syslog ...
# If only_count==1 then the number of messages found is printed to stdout.
function syslog_count_or_check_messages
{
[ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)"
local host=$1 nr_msg_req=$2 only_count=$3
local waited=0 maxwait=$syslog_msg_wait_time
while true; do
local nr_msg_act
nr_msg_act="$(lib_remote_idfile $host \
"egrep '$syslog_err_msg_pattern' $syslog_logfile")" || \
lib_exit 1
nr_msg_act=$(echo "$nr_msg_act" | grep -vw egrep | wc -l)
if [ $only_count -eq 1 ]; then
echo $nr_msg_act
return
fi
if [ $nr_msg_act -eq $nr_msg_req ]; then
break
fi
sleep 1
let waited+=1
lib_vmsg " waited $waited for $nr_msg_req (act. found = $nr_msg_act)"
if [ $waited -eq $maxwait ]; then
lib_exit 1 "maxwait $maxwait exceeded"
fi
done
}

25
test_suite/mars/test_cases/admin/syslog_messages.conf Normal file
View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG
#
# Email: frank.liepold@1und1.de
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#####################################################################
lib_rw_part_of_device_size_written_per_loop=$((1024*1024))
run_list="resource_prepare resource_run_first syslog_run lib_general_mars_checks_after_every_test"

0
test_suite/mars/test_cases/admin/syslog_messages/i_am_a_testdirectory Normal file
View File