From c831e34cecc4177cbfcbbe2fe08afc35182086cb Mon Sep 17 00:00:00 2001 From: Frank Liepold Date: Thu, 13 Mar 2014 14:49:00 +0100 Subject: [PATCH] test_suite: new test case for logging messages in syslogds logfile --- test_suite/mars/default-syslog.conf | 54 ++++++ test_suite/mars/mars_test_cronjob.sh | 1 + test_suite/mars/modules/lib_wait.sh | 10 +- test_suite/mars/modules/syslog.sh | 160 ++++++++++++++++++ .../test_cases/admin/syslog_messages.conf | 25 +++ .../syslog_messages/i_am_a_testdirectory | 0 6 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 test_suite/mars/default-syslog.conf create mode 100644 test_suite/mars/modules/syslog.sh create mode 100644 test_suite/mars/test_cases/admin/syslog_messages.conf create mode 100644 test_suite/mars/test_cases/admin/syslog_messages/i_am_a_testdirectory diff --git a/test_suite/mars/default-syslog.conf b/test_suite/mars/default-syslog.conf new file mode 100644 index 00000000..418281ef --- /dev/null +++ b/test_suite/mars/default-syslog.conf @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +## defaults for module syslog + +## file where the mars messages are logged +syslog_logfile=/var/log/syslog + +## prefix of files containing parameters for logging +syslog_paramfile_prefix=/proc/sys/mars/syslog_flood_ + +## file containing the message class beyond which messages are to be logged +syslog_flood_class_file=${syslog_paramfile_prefix}class + +## file containing the number of messages which are logged within +## the recovery window (for further information see the mars manual) +syslog_flood_limit_file=${syslog_paramfile_prefix}limit + +## file containing the duration of the recovery window +syslog_flood_recovery_s_file=${syslog_paramfile_prefix}recovery_s + +## msg class beyond (incl.) which messages are logged +syslog_flood_class=3 # errors + +## number of messages (see above) +syslog_flood_limit=20 + +## recovery time (see above) +syslog_flood_recovery_s=30 + +## error message pattern to egrep for +syslog_err_msg_pattern='.*(s_trans_logger.*replay error|make_log_finalize.*replay stopped with error)' + +## time to wait for the syslog_msg_number_limit messages to appear +syslog_msg_wait_time=10 + diff --git a/test_suite/mars/mars_test_cronjob.sh b/test_suite/mars/mars_test_cronjob.sh index b9d4ba66..884391d4 100755 --- a/test_suite/mars/mars_test_cronjob.sh +++ b/test_suite/mars/mars_test_cronjob.sh @@ -184,6 +184,7 @@ tests_to_execute=( "test_cases/admin/resizing:test_cases/admin" "test_cases/admin/logrotate:test_cases/admin" "test_cases/admin/logdelete:test_cases/admin" +"test_cases/admin/syslog_messages:test_cases/admin" "test_cases/bugs/memleak:test_cases/bugs" "test_cases/admin/leave_resource_while_sync*:test_cases/admin" "test_cases/admin/switch2primary:test_cases/admin" diff --git a/test_suite/mars/modules/lib_wait.sh b/test_suite/mars/modules/lib_wait.sh index ae737a6c..9980e095 100644 --- a/test_suite/mars/modules/lib_wait.sh +++ b/test_suite/mars/modules/lib_wait.sh @@ -21,7 +21,7 @@ function lib_wait_until_logfile_has_length { [ $# -eq 7 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local host=$1 logfile=$2 length_logfile=$3 - local varname_time_waited=$4 maxwait=$5 check_net_throughput=$6 varname_net_throughput=$7 + local varname_time_waited=$4 maxwait=$5 check_net_throughput=$6 varname_net_throughput="$7" local act_length local waited=0 start_time=$(date +'%s') end_time local my_net_throughput=0 net_throughput_sum=0 net_check_count=0 @@ -68,7 +68,7 @@ function lib_wait_until_fetch_stops [ $# -eq 9 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local module=$1 secondary_host=$2 primary_host=$3 res=$4 local varname_logfile=$5 varname_length_logfile=$6 - local varname_time_waited=$7 check_net_throughput=$8 varname_net_throughput=$9 + local varname_time_waited=$7 check_net_throughput=$8 varname_net_throughput="$9" local maxtime_fetch time_constant_fetch var v for var in maxtime_fetch time_constant_fetch; do @@ -85,7 +85,7 @@ function lib_wait_until_fetch_stops $time_constant_fetch \ $varname_logfile $varname_length_logfile \ $varname_time_waited \ - $check_net_throughput $varname_net_throughput + $check_net_throughput "$varname_net_throughput" } function lib_wait_internal_until_fetch_stops @@ -93,7 +93,7 @@ function lib_wait_internal_until_fetch_stops [ $# -eq 10 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local secondary_host=$1 res=$2 primary_host=$3 maxwait=$4 inactive_wait=$5 local varname_logfile=$6 varname_logfile_length=$7 varname_time_waited=$8 - local check_net_throughput=$9 varname_net_throughput=${10} + local check_net_throughput=$9 varname_net_throughput="${10}" local inactive_waited=0 msg local my_logfile length file_and_length file_and_length_old="x" local waited=0 msg start_time=$(date +'%s') end_time @@ -156,7 +156,7 @@ function lib_wait_until_action_stops [ $# -eq 8 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local action=$1 host=$2 res=$3 maxwait=$4 inactive_wait=$5 local varname_time_waited=$6 check_net_throughput=$7 - local varname_net_throughput=$8 + local varname_net_throughput="$8" local waited=0 link_value link_value_old="x" local inactive_waited=0 msg start_time=$(date +'%s') end_time local link=$(lib_linktree_get_res_host_linkname $host $res $action) diff --git a/test_suite/mars/modules/syslog.sh b/test_suite/mars/modules/syslog.sh new file mode 100644 index 00000000..3f39d727 --- /dev/null +++ b/test_suite/mars/modules/syslog.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +function syslog_run +{ + local primary_host=${global_host_list[0]} + local secondary_host=${global_host_list[1]} + local res=${resource_name_list[0]} + local writer_pid writer_script write_count + local logfile length_logfile time_waited + local logfile_sav=$global_mars_directory/logfile.sav.$$ + local nr_msg_orig i + + lib_wait_for_initial_end_of_sync $primary_host $secondary_host $res \ + $resource_maxtime_initial_sync \ + $resource_time_constant_initial_sync \ + "time_waited" + lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited" + + mount_mount_data_device $primary_host $res + + # 2 loops to test recovery window + for i in 1 2; do + resource_clear_data_device $primary_host $res + + lib_rw_start_writing_data_device $primary_host "writer_pid" \ + "writer_script" 2 2 $res "" + + marsadm_pause_cmd "apply" $secondary_host $res + + lib_wait_until_action_stops "replay" $secondary_host $res \ + $apply_fetch_maxtime_apply \ + $apply_fetch_time_constant_apply \ + "time_waited" 0 "" + lib_vmsg " ${FUNCNAME[0]}: apply time: $time_waited" + + marsview_wait_for_state $secondary_host $res "disk" "Outdated\[.*A.*\]" \ + $marsview_wait_for_state_time + marsview_wait_for_state $secondary_host $res "repl" '-SF--' \ + $marsview_wait_for_state_time || lib_exit 1 + + lib_rw_stop_writing_data_device $primary_host $writer_script \ + "write_count" + main_error_recovery_functions["lib_rw_stop_scripts"]= + + lib_wait_until_fetch_stops "apply_fetch" $secondary_host $primary_host \ + $res "logfile" "length_logfile" \ + "time_waited" 0 "" + lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited" + + file_handling_check_equality_of_file_lengths $logfile $primary_host \ + $secondary_host \ + $length_logfile + + syslog_set_logging_parameters $secondary_host + + lib_vmsg " copying $secondary_host:$logfile to $logfile_sav" + lib_remote_idfile $secondary_host \ + "rm -f $logfile_sav && cp $logfile $logfile_sav" || \ + lib_exit 1 + + file_destroy_dd_on_logfile $secondary_host $logfile $length_logfile + + nr_msg_orig=$(syslog_count_or_check_messages $secondary_host \ + $syslog_flood_limit 1) || lib_exit 1 + + marsadm_do_cmd $secondary_host "resume-replay" $res || lib_exit 1 + + syslog_count_or_check_messages $secondary_host \ + $(( $nr_msg_orig + $syslog_flood_limit )) 0 + + # stopp generation of new error messages + marsadm_pause_cmd "apply" $secondary_host $res + + lib_wait_until_action_stops "replay" $secondary_host $res \ + $apply_fetch_maxtime_apply \ + $apply_fetch_time_constant_apply \ + "time_waited" 0 "" + + lib_vmsg " restoring $logfile from $logfile_sav" + lib_remote_idfile $secondary_host \ + "dd if=$logfile_sav of=$logfile conv=notrunc" || \ + lib_exit 1 + + marsadm_do_cmd $secondary_host "resume-replay" $res || lib_exit 1 + + nr_msg_orig=$(syslog_count_or_check_messages $secondary_host \ + $syslog_flood_limit 1) || lib_exit 1 + + lib_vmsg " sleeping syslog_recovery_s = $syslog_recovery_s seconds" + + syslog_count_or_check_messages $secondary_host $nr_msg_orig 0 + done + +} + +function syslog_set_logging_parameters +{ + local host=$1 param file value varname filename + for param in class limit recovery_s; do + varname='syslog_flood_'$param + lib_vmsg " setting $varname on $host" + eval value='$'$varname + if [ -z "$value" ]; then + lib_exit 1 "no value found for variable $varname" + fi + filename='syslog_flood_'$param'_file' + eval file='$'$filename + lib_remote_idfile $host "ls -l $file" || lib_exit 1 + lib_remote_idfile $host "echo $value > $file" || lib_exit 1 + done +} + +# we cannot use lib_err_wait_for_error_messages, because we need a grep -v egrep +# because sometimes all commands are logged in /var/log/syslog ... +# If only_count==1 then the number of messages found is printed to stdout. +function syslog_count_or_check_messages +{ + [ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 nr_msg_req=$2 only_count=$3 + local waited=0 maxwait=$syslog_msg_wait_time + while true; do + local nr_msg_act + nr_msg_act="$(lib_remote_idfile $host \ + "egrep '$syslog_err_msg_pattern' $syslog_logfile")" || \ + lib_exit 1 + nr_msg_act=$(echo "$nr_msg_act" | grep -vw egrep | wc -l) + if [ $only_count -eq 1 ]; then + echo $nr_msg_act + return + fi + if [ $nr_msg_act -eq $nr_msg_req ]; then + break + fi + sleep 1 + let waited+=1 + lib_vmsg " waited $waited for $nr_msg_req (act. found = $nr_msg_act)" + if [ $waited -eq $maxwait ]; then + lib_exit 1 "maxwait $maxwait exceeded" + fi + done +} diff --git a/test_suite/mars/test_cases/admin/syslog_messages.conf b/test_suite/mars/test_cases/admin/syslog_messages.conf new file mode 100644 index 00000000..562e1bd3 --- /dev/null +++ b/test_suite/mars/test_cases/admin/syslog_messages.conf @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2010-2014 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +lib_rw_part_of_device_size_written_per_loop=$((1024*1024)) + +run_list="resource_prepare resource_run_first syslog_run lib_general_mars_checks_after_every_test" + diff --git a/test_suite/mars/test_cases/admin/syslog_messages/i_am_a_testdirectory b/test_suite/mars/test_cases/admin/syslog_messages/i_am_a_testdirectory new file mode 100644 index 00000000..e69de29b