diff --git a/test_suite/bugs.txt b/test_suite/bugs.txt index 10c1ffda..d480e452 100644 --- a/test_suite/bugs.txt +++ b/test_suite/bugs.txt @@ -43,3 +43,18 @@ - 'size' link is actual state - no target state + +*5. memleak: marsadm secondary produces memleaks if executed during "flying writes" + (test /home/fl/mars/test_suite/test_cases/bugs/memleak) + Fixed: + commit 281fa300b27d574b2c11066bf2c4f6114ade1325 + Author: Frank Liepold 2013-09-16 12:53:22 + Branch: master + Follows: light0.1beta0.12 + Precedes: + + light: workaround flying IO before reporting memory leaks + + +6. marsadm invalidate does not repair secondary after mars dir full + (test /home/fl/mars/test_suite/test_cases/hardcore/mars_dir_full) diff --git a/test_suite/build_test_environment/checkout/checkout_mars_from_git.conf b/test_suite/build_test_environment/checkout/checkout_mars_from_git.conf index 4ff95862..bcd65193 100644 --- a/test_suite/build_test_environment/checkout/checkout_mars_from_git.conf +++ b/test_suite/build_test_environment/checkout/checkout_mars_from_git.conf @@ -1,7 +1,7 @@ #!/bin/bash verbose_script=1 checkout_mars_src_directory=/home/fl/mars/mars-testing -checkout_mars_git_branch=master +checkout_mars_git_branch=WIP-transferstatus checkout_mars_git_tag= checkout_mars_kernel_src_directory=/home/fl/mars/linux-infong-3.2 checkout_mars_kernel_git_branch=infong-3.2.47-tst-1 diff --git a/test_suite/build_test_environment/lv_config/lv_recreate.conf b/test_suite/build_test_environment/lv_config/lv_recreate.conf index 3f3c92e5..feb13913 100644 --- a/test_suite/build_test_environment/lv_config/lv_recreate.conf +++ b/test_suite/build_test_environment/lv_config/lv_recreate.conf @@ -18,5 +18,5 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. verbose_script=1 -prepare_list="resource_leave_all lv_config_prepare" +prepare_list="lv_config_prepare" run_list="lv_config_run" diff --git a/test_suite/changes.dok b/test_suite/changes.dok index c2d37e1a..148b0ef8 100644 --- a/test_suite/changes.dok +++ b/test_suite/changes.dok @@ -22,7 +22,9 @@ Dateien Doku: - Alle globalen Variablen in .conf - Dateien - Die Reihenfolge der default-*.conf - Dateien ist u.U. wichtig, wenn die - gegenseitig Variable referenziert werden!! + gegenseitig Variable referenziert werden!! Daher sollte hier eine + entsprechende _set_globals Funktion definiert sein (siehe z.B. + resource_set_globals) - Alle globalen Variablen und alle Funktionen mit Modulname als Präfix - Globale Arrays muessen mit declare -g deklariert werden, da die Variablen via source innerhalb einer Funktion bekannt gemacht werden diff --git a/test_suite/default-cluster.conf b/test_suite/default-cluster.conf index 320b3554..f3707328 100644 --- a/test_suite/default-cluster.conf +++ b/test_suite/default-cluster.conf @@ -17,5 +17,5 @@ declare -g -A cluster_mars_dir_lv_name_list eval cluster_mars_dir_lv_name_list=($(x=(${main_host_list[@]/#/[}); y=(${x[@]/%/]=lv-6-100}); echo ${y[@]})) # -cluster_debugfiles=("$main_mars_directory/5.total.log" ) +cluster_debugfile="$main_mars_directory/5.total.log" diff --git a/test_suite/default-lib_rw.conf b/test_suite/default-lib_rw.conf index d8369c02..1238a456 100644 --- a/test_suite/default-lib_rw.conf +++ b/test_suite/default-lib_rw.conf @@ -18,4 +18,8 @@ lib_rw_file_to_write="file_to_write" lib_rw_write_and_delete_script="/tmp/write_and_delete.sh.$$" +## part of data device size, which is written in the dd statement in the +## script wich writes the data device +lib_rw_part_of_device_size_written_per_loop=4 + lib_rw_number_of_umount_retries_after_stopped_write=4 diff --git a/test_suite/default-lv_config.conf b/test_suite/default-lv_config.conf index 12f443af..9bd747f3 100644 --- a/test_suite/default-lv_config.conf +++ b/test_suite/default-lv_config.conf @@ -18,19 +18,23 @@ ## Only logical volumes which do not exist or which have a wrong size will ## be recreated. -## the names are built due to the pattern lv--, where i runs from -## 0 to $resource_count - 1 +## the names are built due to the pattern lv-- lv_config_name_list=(lv-1-2 lv-2-2 lv-3-2 lv-4-2 lv-5-10 lv-6-100) -lv_config_min_lvg_size=200 # GB +lv_config_min_lvg_size=100 # GB lv_config_host_list[0]="no_target_hosts_given" -## lv_config_delete_lvs_not_needed -## -## if set, all logical volumes belonging to volume group main_lvg_name will -## be deleted. - -lv_config_delete_lvs_not_needed=1 +declare -g -A lv_config_partition_count_list +lv_config_partition_count_list=(\ + [istore-test-bs7]=3 \ + [istore-test-bap7]=3 \ + ) +declare -g -A lv_config_partition_list +lv_config_partition_list=(\ + [istore-test-bs7]="$(eval echo /dev/cciss/c1d{0..${lv_config_partition_count_list["istore-test-bs7"]}})" \ + [istore-test-bap7]="$(eval echo /dev/cciss/c1d{0..${lv_config_partition_count_list["istore-test-bap7"]}})" \ + ) +lv_config_stripesize=64K ## option for mkfs. call declare -g -A lv_config_mkfs_option_list @@ -39,5 +43,5 @@ lv_config_mkfs_option_list=([xfs]="-f" [ext3]="" [ext4]="") ## filesystem specific tune commands ## the string will be replaced by the actual device name declare -g -A lv_config_fs_type_tune_cmd_list -lv_config_fs_type_tune_cmd_list=([xfs]="" [ext3]="tune -c 0 " [ext4]="tune -c 0 ") +lv_config_fs_type_tune_cmd_list=([xfs]="" [ext3]="tune2fs -c 0 " [ext4]="tune2fs -c 0 ") diff --git a/test_suite/default-main.conf b/test_suite/default-main.conf index fb6af7bd..d371fbab 100644 --- a/test_suite/default-main.conf +++ b/test_suite/default-main.conf @@ -3,18 +3,21 @@ ## the base directory of all .sh and .conf files main_base_directory=/home/fl/mars/test_suite -## hosts the tests are running on +## hosts the tests are running on. The first host is initially always used as +## primary host main_host_list=("istore-test-bs7" "istore-test-bap7") -## using lilo for all hosts -## (constructing the expression ([host1]=lilo [host2]=lilo)) declare -g -A main_host_bootloader_list -eval main_host_bootloader_list=($(x=(${main_host_list[@]/#/[}); y=(${x[@]/%/]=lilo}); echo ${y[@]})) +main_host_bootloader_list=(\ + [istore-test-bs7]=grub \ + [istore-test-bap7]=lilo \ + ) -## using lilo label mars for all hosts declare -g -A main_host_bootloader_label_list -eval main_host_bootloader_label_list=($(x=(${main_host_list[@]/#/[}); y=(${x[@]/%/]=mars}); echo ${y[@]})) - +main_host_bootloader_label_list=(\ + [istore-test-bs7]=no_label \ + [istore-test-bap7]=mars \ + ) ## associative array containing functions (as indexes) to be called in case of ## unexpected errors or signals. See also lib_exit diff --git a/test_suite/default-memleak.conf b/test_suite/default-memleak.conf new file mode 100644 index 00000000..e69de29b diff --git a/test_suite/default-multires.conf b/test_suite/default-multires.conf new file mode 100644 index 00000000..a4de7bf7 --- /dev/null +++ b/test_suite/default-multires.conf @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### +verbose_script=1 + +## size of logical volumes to be created +multires_lv_size=700 + +## number of logical volumes to be created +multires_lv_count=8 + diff --git a/test_suite/default-net.conf b/test_suite/default-net.conf index 831163f6..80953b0a 100644 --- a/test_suite/default-net.conf +++ b/test_suite/default-net.conf @@ -41,7 +41,7 @@ net_time_constant_apply=5 ## time for which the amount of data to fetch must be constant to declare ## the fetch process as having stopped -net_time_constant_fetch=10 +net_time_constant_fetch=20 ## maxtime to wait for apply to stop (after pause-apply) net_maxtime_apply=60 diff --git a/test_suite/default-perftest.conf b/test_suite/default-perftest.conf new file mode 100644 index 00000000..37918d22 --- /dev/null +++ b/test_suite/default-perftest.conf @@ -0,0 +1,152 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +## defaults for module perftest + +## time for which the amount of data to sync must be constant +## to declare the sync process as inactive +perftest_time_constant_sync=20 + +## maxtime to wait for sync to stop +perftest_maxtime_sync=300 + +## flag whether the data device should be written simultaneously +perftest_parallel_writer=0 + +## amount of data in GB to be written to the data device +perftest_data_in_gb_to_write=10 + +## size of logfile (in GiB and rounded) after perftest_data_in_gb_to_write are written +perftest_logfile_size_in_gb=10 + +## time for which the amount of data to fetch must be constant to declare +## the fetch process as having stopped +perftest_time_constant_fetch=10 + +## maxtime to wait for fetch to stop (after pause-fetch) +perftest_maxtime_fetch=300 + +## maxtime to wait for apply to stop (after pause-apply) +perftest_maxtime_apply=100 + +## time for which the amount of data to apply must be constant to declare +## the apply process as having stopped +perftest_time_constant_apply=5 + +## error tag which will be printed if the runtime diff exceeds the +## required interval +perftest_errortag_result_out_of_bounds="PERFORMANCE-FAILURE" + +## maxtime to wait for secondary to become disk state = Uptodate and +## repl state = Replaying in *prepare_for* functions +perftest_maxtime_state_constant=60 + +## array containing the required results (durations or rate (loops / minute)) of performance tests. A result consists of the +## required value and - comma separated - a tolerance given in percent. +## the index consists of host, action, flag whether a parallel writer to the data device is running, type of result, +## number of resources, sub testcase id +## +## for the sync tests the sub testcase id consist of: nunber of patches, patch length, sync mode +declare -g -A perftest_required_result_list +perftest_required_result_list=(\ + ["HOST","ACTION","PARALLEL_WRITER","RESULT_TYPE","NO_RESOURCES","GB_TO_FETCH_RESP_TO_APPLY"]=-1 \ + + ["istore-test-bap7","apply",0,"time",1,"10"]=63,10 \ + ["istore-test-bap7","apply",1,"time",1,"10"]=63,10 \ + + ["istore-test-bap7","fetch",0,"time",1,"10"]=155,10 \ + ["istore-test-bap7","fetch",1,"time",1,"10"]=223,10 \ + + ["istore-test-bap7","fetch_and_apply",0,"time",1,"5"]=267,10 \ + ["istore-test-bap7","fetch_and_apply",1,"time",1,"5"]=336,10 \ + + + ["HOST","ACTION","PARALLEL_WRITER","RESULT_TYPE","NO_RESOURCES","GB_FETCHED_RESP_APPLIED"]=-1 \ + + ["istore-test-bs7","write_while_apply",0,"loops_per_min",1,"10"]=16,10 \ + ["istore-test-bs7","write_while_fetch",0,"loops_per_min",1,"10"]=16,10 \ + ["istore-test-bs7","write_while_fetch_and_apply",0,"loops_per_min",1,"5"]=15,10 \ + + + ["HOST","ACTION","PARALLEL_WRITER","RESULT_TYPE","NO_RESOURCES","DURATION_OF_WRITE:DEVICE_SIZE:PART_OF_DEVICE_SIZE_WRITTEN_PER_LOOP"]=-1 \ + + ["istore-test-bs7","write",0,"loops_per_min",1,"30:2:4"]=24,10 \ + + ["HOST","ACTION","PARALLEL_WRITER","RESULT_TYPE","NO_RESOURCES","DURATION_OF_WRITE:DEVICE_SIZE:PART_OF_DEVICE_SIZE_WRITTEN_PER_LOOP:DIVISION_OF_DATA_AND_MARS_DEVICES"]=-1 \ + + ["istore-test-bap7","write",0,"loops_per_min",1,"60:2:8:same_controller"]=28,10 \ +["istore-test-bap7","write",0,"loops_per_min",1,"60:2:8:separated_and_mars_dev_with_bbu_cache"]=22,10 \ +["istore-test-bap7","write",0,"loops_per_min",1,"60:2:8:separated_and_mars_dev_without_bbu_cache"]=10,10 \ + + + ["HOST","ACTION","PARALLEL_WRITER","RESULT_TYPE","NO_RESOURCES","NO_PATCHES:PATCH_LENGTH:SYNC_MODE"]=-1 \ + + ["istore-test-bap7","sync",0,"time",1,"10:1:rsync"]=42,10 \ + ["istore-test-bap7","sync",0,"time",1,"10:50000:rsync"]=43,10 \ + ["istore-test-bap7","sync",0,"time",1,"4:500000:rsync"]=45,10 \ + ["istore-test-bap7","sync",0,"time",1,"2:1000000:rsync"]=49,10 \ + + ["istore-test-bap7","sync",0,"time",1,"10:1:fast_sync"]=10,10 \ + ["istore-test-bap7","sync",0,"time",1,"10:50000:fast_sync"]=10,10 \ + ["istore-test-bap7","sync",0,"time",1,"4:500000:fast_sync"]=13,10 \ + ["istore-test-bap7","sync",0,"time",1,"2:1000000:fast_sync"]=18,10 \ + + ["istore-test-bap7","sync",1,"time",1,"10:1:fast_sync"]=33,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"10:1:fast_sync"]=11,10 \ + ["istore-test-bap7","sync",1,"time",1,"10:50000:fast_sync"]=83,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"10:50000:fast_sync"]=12,10 \ + ["istore-test-bap7","sync",1,"time",1,"4:500000:fast_sync"]=60,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"4:500000:fast_sync"]=10,10 \ + ["istore-test-bap7","sync",1,"time",1,"2:1000000:fast_sync"]=56,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"2:1000000:fast_sync"]=10,10 \ + + ["istore-test-bap7","sync",0,"time",1,"10:1:no_fast_sync"]=26,10 \ + ["istore-test-bap7","sync",0,"time",1,"10:50000:no_fast_sync"]=40,10 \ + ["istore-test-bap7","sync",0,"time",1,"4:500000:no_fast_sync"]=82,10 \ + ["istore-test-bap7","sync",0,"time",1,"2:1000000:no_fast_sync"]=31,10 \ + + ["istore-test-bap7","sync",1,"time",1,"10:1:no_fast_sync"]=31,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"10:1:no_fast_sync"]=10,10 \ + ["istore-test-bap7","sync",1,"time",1,"10:50000:no_fast_sync"]=69,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"10:50000:no_fast_sync"]=10,10 \ + ["istore-test-bap7","sync",1,"time",1,"4:500000:no_fast_sync"]=71,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"4:500000:no_fast_sync"]=10,10 \ + ["istore-test-bap7","sync",1,"time",1,"2:1000000:no_fast_sync"]=78,10 \ + ["istore-test-bs7","write_while_sync",0,"loops_per_min",1,"2:1000000:no_fast_sync"]=10,10 \ + ) + +## flag to check net throughput while waiting for fetch or sync to stop +perftest_check_net_throughput=1 + +## flag whether the performance influence of splitting /mars device and data +## device on different raid controllers is to be measured +perftest_division_mars_device_data_device=0 + +## after this number of checks whether fetch or sync have stopped, the net throughput is checked +perftest_check_net_throughput_intervall=2 + +## port on which nttcp tests the network connection +perftest_nttcp_port=12346 + +## command to start nttcp receiver +perftest_nttcp_start_cmd="nttcp -p $perftest_nttcp_port -r -i" + +## type of result of tests +perftest_result_type="time" + diff --git a/test_suite/default-resource.conf b/test_suite/default-resource.conf index d719c081..c70b8be3 100644 --- a/test_suite/default-resource.conf +++ b/test_suite/default-resource.conf @@ -12,9 +12,6 @@ ## ## this module administrates the test resources -## number of resources -resource_count="6" - ## the names of the resources used in the current test resource_name_list=(lv-1-2) @@ -22,16 +19,30 @@ resource_name_list=(lv-1-2) ## maps resource name lv-- to /mnt/mars-test- declare -g -A resource_mount_point_list -eval resource_mount_point_list=($(for i in ${!resource_name_list[*]};do printf "[${resource_name_list[$i]}]=/mnt/mars-test-$(($i + 1)) ";done)) ## resource dir of resource res is given by ## $main_mars_directory/resource-$res declare -g -A resource_dir_list -eval resource_dir_list=($(for i in ${!resource_name_list[*]};do printf "[${resource_name_list[$i]}]=$main_mars_directory/resource-${resource_name_list[$i]} ";done)) ## filesystem type of resource devices is set to xfs declare -g -A resource_fs_type_list -eval resource_fs_type_list=($(for i in ${!resource_name_list[*]};do printf "[${resource_name_list[$i]}]=xfs ";done)) + +function resource_set_globals +{ + + eval resource_mount_point_list=($(for i in ${!resource_name_list[*]};do printf "[${resource_name_list[$i]}]=/mnt/mars-test-$(($i + 1)) ";done)) + + eval resource_dir_list=($(for i in ${!resource_name_list[*]};do printf "[${resource_name_list[$i]}]=$main_mars_directory/resource-${resource_name_list[$i]} ";done)) + + eval resource_fs_type_list=($(for i in ${!resource_name_list[*]};do printf "[${resource_name_list[$i]}]=xfs ";done)) + + ## messages to be grepped for in abnormal situations + resource_mars_dir_full_err_pattern_list[0]='DISK SPACE IS EXTREMELY LOW on '"${resource_dir_list[${resource_name_list[0]}]}" + + resource_mars_dir_full_err_pattern_list[1]='EMERGENCY MODE on '"${resource_dir_list[${resource_name_list[0]}]}"'.*stopped transaction logging.*created a hole in the logfile sequence' +} + +resource_set_globals ## mounting of the data device must fail on a secondary resource. Because @@ -75,9 +86,7 @@ resource_big_file=$main_mars_directory/mars_test_bigfile declare -g -A resource_msgfile_list resource_msgfile_list=([err]='3.error.status' [warn]='2.warn.status') -## messages to be grepped for in abnormal situations -resource_mars_dir_full_err_pattern_list[0]='DISK SPACE IS EXTREMELY LOW on '"${resource_dir_list[${resource_name_list[0]}]}" - -resource_mars_dir_full_err_pattern_list[1]='EMERGENCY MODE on '"${resource_dir_list[${resource_name_list[0]}]}"'.*stopped transaction logging.*created a hole in the logfile sequence' +## file used to reset after emergency mode +resource_proc_sys_mars_reset_emergency_file="/proc/sys/mars/mars_reset_emergency" resource_mars_dir_full_warn_pattern_list[0]='EMERGENCY: the space on '$main_mars_directory'/ is very low.' diff --git a/test_suite/default-synctest.conf b/test_suite/default-synctest.conf deleted file mode 100644 index febd58eb..00000000 --- a/test_suite/default-synctest.conf +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG -# -# Email: frank.liepold@1und1.de -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -## defaults for module synctest - -## list of lengths of the patch (in KB) the data device will be patched with -synctest_patch_length_list=(1 50000 500000 1000000) - -## number of patches the data device will be patched with -## must correspond to synctest_patch_length_list -synctest_number_of_patches_list=(10 10 4 2) - -## time for which the amount of data to sync must be constant -## to declare the sync process as inactive -synctest_time_constant_sync=20 - -## maxtime to wait for sync to stop -synctest_maxtime_sync=300 - -## sync modus -synctest_fast_sync=1 - -## proc file to switch sync modus -synctest_sync_mode_proc_file='/proc/sys/mars/do_fast_fullsync' - -## use rsync instead of mars sync -synctest_use_mars_sync=1 - -## data file to user for rsync -synctest_data_file=$main_mars_directory/file_to_sync - -## flag whether the data device should be written while syncing -synctest_parallel_writer=0 - -## time to run the data device writer to determine write rate -synctest_write_time=10 - -## time for which the amount of data to fetch must be constant to declare -## the fetch process as having stopped -synctest_time_constant_fetch=10 - -## maxtime to wait for fetch to stop (after pause-fetch) -synctest_maxtime_fetch=300 - -## maxtime to wait for apply to stop (after pause-apply) -synctest_maxtime_apply=100 - -## time for which the amount of data to apply must be constant to declare -## the apply process as having stopped -synctest_time_constant_apply=5 diff --git a/test_suite/mars_test_cronjob.sh b/test_suite/mars_test_cronjob.sh index a49424f2..2f59c89b 100755 --- a/test_suite/mars_test_cronjob.sh +++ b/test_suite/mars_test_cronjob.sh @@ -24,11 +24,29 @@ function execute_tests fi done + local error_ocurred=0 + local msg + local grep_cmd='grep PERFORMANCE-FAILURE '$logfile if [ ${#failed_tests[*]} -ne 0 ];then - local to msg - msg="tests failed on $(hostname) (Script $0): - ${failed_tests[@]} -for details see $logfile on $(hostname)" + error_ocurred=1 + local msg + msg="tests failed on $(hostname) (Script $0):"$'\n' + msg+=$(echo ${failed_tests[@]} | tr " " $'\n')$'\n' + fi + if $grep_cmd >/dev/null; then + error_ocurred=1 + msg+=$'\n'"Performance-Failures:"$'\n' + msg+="$($grep_cmd)"$'\n' + fi + grep_cmd='grep ERROR-FILE '$logfile + if $grep_cmd >/dev/null; then + error_ocurred=1 + msg+=$'\n'"Error-Files:"$'\n' + msg+="$($grep_cmd)"$'\n' + fi + if [ $error_ocurred -eq 1 ]; then + local to + msg+="for details see $logfile on $(hostname)" for to in "${mail_to[@]}"; do sendEmail -m "$msg" -f $mail_from -t $to -u "failed mars tests" -s $mail_server done @@ -68,7 +86,7 @@ done # main echo Start $(basename $0) at $(date) -logfile="/home/fl/tmp/cronjob_mars" +logfile="/home/fl/tmp/cronjob_mars.log" eval $(ssh-agent) ~/tools/sx @@ -95,6 +113,7 @@ test_cases/destroy_secondary_logfile \ test_cases/admin/resizing \ test_cases/admin/logrotate \ test_cases/admin/logdelete \ +test_cases/bugs/memleak \ test_cases/perf \ test_cases/admin/switch2primary \ test_cases/admin/datadev_full \ @@ -105,7 +124,7 @@ test_cases/stabil/crash/crash_primary \ test_cases/stabil/crash/crash_primary_logger_comletion_semantics__aio_sync_mode \ test_cases/stabil/crash/crash_primary_logger_completion_semantics \ test_cases/stabil/crash/crash_primary_aio_sync_mode \ -test_cases/hardcore/aio_filehandle \ +test_cases/bugs/aio_filehandle \ build_test_environment/resource/leave_resource \ ) diff --git a/test_suite/scripts/modules/20_install_mars.sh b/test_suite/scripts/modules/20_install_mars.sh index f1c19cbb..670b2dd8 100644 --- a/test_suite/scripts/modules/20_install_mars.sh +++ b/test_suite/scripts/modules/20_install_mars.sh @@ -106,7 +106,7 @@ function install_mars_update_bootloader_on_target_hosts lib_remote_idfile $host lilo || lib_exit 1 install_mars_activate_kernel_to_boot_with_lilo $host $label_name ;; - *) lib_exit 1 "bootloader $boot_loader not supported" + *) echo "hint: for bootloader $boot_loader on $host no action defined" ;; esac done diff --git a/test_suite/scripts/modules/23_lv_config.sh b/test_suite/scripts/modules/23_lv_config.sh index c0a738de..cae878bc 100644 --- a/test_suite/scripts/modules/23_lv_config.sh +++ b/test_suite/scripts/modules/23_lv_config.sh @@ -12,29 +12,53 @@ function lv_config_name_matches_our_list return 1 } +function lv_exists +{ + local host=$1 lv_name=$2 + local dev=$(lv_config_get_lv_device $lv_name) + lib_vmsg " checking whether lv $lv_name (dev=$dev) on $host exists" + lib_remote_idfile $host "lvdisplay --noheadings -C $dev -o lv_name" + return $? +} + function lv_config_prepare { lv_config_check_variables - if (( $lv_config_delete_lvs_not_needed )); then - local host - for host in "${main_host_list[@]}"; do - local lv_name_list - lv_name_list=$(lib_remote_idfile $host \ - lvdisplay $main_lvg_name -C --noheadings -o lv_name) \ - || lib_exit 1 - local lv_name - for lv_name in ${lv_name_list[@]}; do - if lv_config_name_matches_our_list $lv_name; then - continue - else - lib_vmsg " deleting lv $main_lvg_name/$lv_name on $host" - lib_remote_idfile $host \ - lvremove -f $main_lvg_name/$lv_name || lib_exit 1 - fi - done - done - fi + resource_leave_all + cluster_umount_mars_dir_all + lv_config_delete_vg } + +function lv_config_delete_vg +{ + local host out i rc + for host in "${main_host_list[@]}"; do + lib_vmsg " removing lvs in $main_lvg_name on $host" + out="$(lib_remote_idfile $host lvdisplay -c $main_lvg_name)" + rc=$? + # rc=5 means: vg does not exist + if [ $rc -ne 5 -a $rc -ne 0 ]; then + lib_exit 1 + fi + for i in $(echo "$out" | awk -F':' '$2=="'$main_lvg_name'"{print $1}') + do + lib_vmsg "lvremove $host:$i" + lib_remote_idfile $host lvremove -f $i || lib_exit 1 + done + lib_vmsg " removing pvs in $main_lvg_name on $host" + out="$(lib_remote_idfile $host pvdisplay -c)" || lib_exit 1 + for i in $(echo "$out" | \ + awk -F':' '$3=="'$main_lvg_name'"{print $1":"$2} + $2=="'$main_lvg_name'"{print $1}' + ) + do + lib_vmsg " removing pvs $host:$i" + lib_remote_idfile $host pvremove -ff -y $i || lib_exit 1 + done + + done +} + function lv_config_extract_int_from_lv_size { @@ -44,29 +68,34 @@ function lv_config_extract_int_from_lv_size function lv_config_check_volume_group_existence_and_size { - local host lvg_size + local host lvg_size rc for host in "${main_host_list[@]}"; do lib_vmsg " checking volume group $main_lvg_name on $host" lvg_size=$(lib_remote_idfile $host vgs --noheadings \ - --units G -o vg_size $main_lvg_name) || lib_exit 1 + --units G -o vg_size $main_lvg_name) + rc=$? + if [ $rc -ne 0 ];then + lib_vmsg " vg $host:$main_lvg_name will be created" + return + fi # 11.1G -> 11 lvg_size=$(lv_config_extract_int_from_lv_size $lvg_size) [ "$lvg_size" -ge $lv_config_min_lvg_size ] || \ - lib_exit 1 "size $lvg_size of volume group $main_lvg_name not >= 100" + lib_exit 1 "size $lvg_size of volume group $main_lvg_name not >= $lv_config_min_lvg_size" done } function lv_config_check_variables { - if [ $resource_count -eq 0 ]; then + if [ ${#lv_config_name_list[*]} -eq 0 ]; then lib_exit 1 "number of logical volumes to be created = 0" fi local lv_name sum for lv_name in ${lv_config_name_list[@]}; do let sum=$(($sum + $(lv_config_get_lv_size $lv_name))) done - if [ $sum -gt $lv_config_min_lvg_size ];then - lib_exit 1 "sum of sizes in lv_config_name_list = $sum exceeds $lv_config_min_lvg_size" + if [ $sum -le $lv_config_min_lvg_size ];then + lib_exit 1 "sum of sizes in lv_config_name_list = $sum smaller than $lv_config_min_lvg_size" fi lib_check_access_to_remote_hosts "$main_ssh_idfile_opt" \ @@ -103,14 +132,11 @@ function lv_config_get_size_logical_volume echo $lv_size } -# lv_config_delete_lvs_not_needed=1 -# -# lvdisplay /dev/vg-mars/huhu - function lv_config_recreate_logical_volumes { local host lv_name lv_size lv_dev lv_size_act rc lv_must_be_recreated local lv_size_tolerance lv_size_diff + lv_config_create_vg for host in "${main_host_list[@]}"; do for lv_name in "${lv_config_name_list[@]}"; do lv_must_be_recreated=0 @@ -132,15 +158,39 @@ function lv_config_recreate_logical_volumes fi fi if (( lv_must_be_recreated )); then - lib_vmsg " creating lv $lv_name (size $lv_size G) on $host" - lib_remote_idfile $host \ - lvcreate -L ${lv_size}G -n $lv_name $main_lvg_name \ - || lib_exit 1 + lv_config_create_lv $host $lv_name fi done done } +function lv_config_create_lv +{ + local host=$1 lv_name=$2 + local size=$(lv_config_get_lv_size $lv_name) + lib_vmsg " creating lv $lv_name (size $size G) on $host" + lib_remote_idfile $host \ + lvcreate -n $lv_name \ + -i ${lv_config_partition_count_list[$host]} \ + -I $lv_config_stripesize -L ${size}G $main_lvg_name \ + || lib_exit 1 +} + +function lv_config_create_vg +{ + local host + for host in "${main_host_list[@]}"; do + local lv + local partitions="${lv_config_partition_list[$host]}" + lib_vmsg " creating $main_lvg_name on $host (partitions=$partitions)" + lib_remote_idfile $host vgcreate $main_lvg_name $partitions \ + || lib_exit 1 + for lv in ${lv_config_name_list[@]}; do + lv_config_create_lv $host $lv + done + done +} + function lv_config_get_lv_size { local lv_name=$1 @@ -158,3 +208,15 @@ function lv_config_run { lv_config_recreate_logical_volumes } + +function lv_config_get_dm_dev +{ + local host=$1 dev=$2 + local dm_dev rc + dm_dev=$(lib_remote_idfile $host "dmsetup info -C --noheadings -o name $dev") + rc=$? + if [ $rc -ne 0 ]; then + return $rc + fi + lib_remote_idfile $host "ls /dev/mapper/$dm_dev" || lib_exit 1 +} diff --git a/test_suite/scripts/modules/34_cluster.sh b/test_suite/scripts/modules/34_cluster.sh index 606000f2..b47d2a01 100644 --- a/test_suite/scripts/modules/34_cluster.sh +++ b/test_suite/scripts/modules/34_cluster.sh @@ -30,7 +30,7 @@ function cluster_umount_data_device_all function cluster_rmmod_mars_all { local host - mount_umount_data_device + mount_umount_data_device_all for host in "${main_host_list[@]}"; do cluster_rmmod_mars $host done @@ -46,6 +46,9 @@ function cluster_rmmod_mars function cluster_clear_mars_dir_all { local host + if [ -z "$main_mars_directory" ]; then + lib_exit 1 " variable main_mars_directory empty" + fi for host in "${main_host_list[@]}"; do lib_vmsg " clearing directory $host:$main_mars_directory" lib_remote_idfile $host "rm -rf $main_mars_directory/*" || lib_exit 1 @@ -63,27 +66,48 @@ function cluster_insert_mars_module_all function cluster_insert_mars_module { local host=$1 + cluster_mount_mars_dir $host cluster_create_debugfiles $host lib_vmsg " inserting mars module on $host" lib_remote_idfile $host 'grep -w "^mars" /proc/modules || modprobe mars' || lib_exit 1 } +function cluster_umount_mars_dir_all +{ + local host + for host in "${main_host_list[@]}"; do + if mount_is_dir_mountpoint $host $main_mars_directory; then + mount_umount $host "device_does_not_matter" $main_mars_directory + fi + done +} + function cluster_mount_mars_dir { - local host=$1 dev=$2 - if mount_is_dir_mountpoint $host $main_mars_directory - then - mount_umount $host "device_does_not_matter" $main_mars_directory + local host=$1 + local dev="$(lv_config_get_lv_device ${cluster_mars_dir_lv_name_list[$host]})" + local already_mounted_correctly=0 + if mount_is_dir_mountpoint $host $main_mars_directory; then + local mount_point + if mount_is_device_mounted $host $dev "mount_point" \ + && [ "$mount_point" == "$main_mars_directory" ] + then + already_mounted_correctly=1 + else + mount_umount $host "device_does_not_matter" $main_mars_directory + fi + fi + if [ $already_mounted_correctly -eq 0 ];then + lib_remote_check_device_fs $host $dev $main_mars_fs_type + mount_mount $host $dev $main_mars_directory $main_mars_fs_type || lib_exit 1 fi - mount_mount $host $dev $main_mars_directory $main_mars_fs_type || lib_exit 1 } function cluster_mount_mars_dir_all { local host dev for host in "${main_host_list[@]}"; do - dev="$(lv_config_get_lv_device ${cluster_mars_dir_lv_name_list[$host]})" - cluster_mount_mars_dir $host $dev + cluster_mount_mars_dir $host done } @@ -197,15 +221,15 @@ function cluster_check_devices_all function cluster_create_debugfiles { local host=$1 - lib_vmsg " creating debugfiles ${cluster_debugfiles[@]}" - lib_remote_idfile $host "touch ${cluster_debugfiles[@]}" || lib_exit 1 + lib_vmsg " creating debugfile $cluster_debugfile" + lib_remote_idfile $host "touch $cluster_debugfile" || lib_exit 1 } function cluster_remove_debugfiles { local host=$1 - lib_vmsg " removing debugfiles ${cluster_debugfiles[@]}" - lib_remote_idfile $host "rm -f ${cluster_debugfiles[@]}" || lib_exit 1 + lib_vmsg " removing debugfiles $cluster_debugfile" + lib_remote_idfile $host "rm -f $cluster_debugfile" || lib_exit 1 } function cluster_check_variables diff --git a/test_suite/scripts/modules/38_resource.sh b/test_suite/scripts/modules/38_resource.sh index 51a720fe..06c68acd 100644 --- a/test_suite/scripts/modules/38_resource.sh +++ b/test_suite/scripts/modules/38_resource.sh @@ -29,8 +29,8 @@ function resource_prepare function resource_check_variables { - if ! expr "${lv_config_name_list[*]}" : "${resource_name_list[*]}" \ - >/dev/null + if ! expr "${lv_config_name_list[*]}" \ + : "\(\(.* \)*${resource_name_list[*]}\( .*\)*\$\)" >/dev/null then lib_exit 1 "resource_name_list = '${resource_name_list[*]}' is no substring of '${lv_config_name_list[*]}' = lv_config_name_list" fi @@ -68,7 +68,7 @@ function resource_leave { local host=$1 res=$2 if resource_is_data_device_mounted $host $res; then - local dev=$(resource_get_name_data_device $res) + local dev=$(resource_get_data_device $res) mount_umount $host $dev ${resource_mount_point_list[$res]} || lib_exit 1 fi @@ -80,7 +80,7 @@ function resource_leave done resource_do_after_leave_loops $host $res fi - resource_rm_resource_dir $res + resource_rm_resource_dir_all $res } function resource_do_after_leave_loops @@ -180,13 +180,13 @@ function resource_multi_res_run : } -function resource_fill_data_device +function resource_fill_mars_dir { local primary_host=${main_host_list[0]} local secondary_host=${main_host_list[1]} local res=${resource_name_list[0]} local dev=$(lv_config_get_lv_device $res) - local data_dev=$(resource_get_name_data_device $res) + local data_dev=$(resource_get_data_device $res) local data_dev_size=$(lv_config_get_lv_size $res) local mars_lv_name=${cluster_mars_dir_lv_name_list[$primary_host]} local mars_dev=$(lv_config_get_lv_device $mars_lv_name) @@ -214,11 +214,18 @@ function resource_fill_data_device lib_remote_idfile $primary_host "rm -f $resource_big_file" || lib_exit 1 fi - resource_recreate_all + resource_check_proc_sys_mars_emergency_file $primary_host + + resource_resize_mars_dir $primary_host $mars_dev $(($mars_dev_size + 10)) + cluster_insert_mars_module $primary_host +echo TODO; return + marsadm_do_cmd $host "down" $res + marsadm_do_cmd $host "invalidate" $res lib_wait_for_initial_end_of_sync $secondary_host $res \ $resource_maxtime_initial_sync \ $resource_time_constant_initial_sync \ "time_waited" + # TODO bischen was schreiben lib_rw_compare_checksums $primary_host $secondary_host $dev 0 \ "primary_cksum" "secondary_cksum" if [ $resource_use_data_dev_writes_to_fill_mars_dir -eq 1 ]; then @@ -229,6 +236,27 @@ function resource_fill_data_device fi } +function resource_resize_mars_dir +{ + local host=$1 mars_dev=$2 new_size=$3 + lib_vmsg " resizing $host:$mars_dev to $new_size GB" + + cluster_rmmod_mars $host + lv_config_resize_device $host $mars_dev $new_size + lib_remote_idfile $host "resize2fs $mars_dev" || lib_exit 1 +} + +function resource_check_proc_sys_mars_emergency_file +{ + local host=$1 value + lib_vmsg " checking value in $host:$resource_proc_sys_mars_reset_emergency_file" + value=$(lib_remote_idfile $host \ + "cat $resource_proc_sys_mars_reset_emergency_file") || lib_exit 1 + if [ $value -ne 1 ];then + lib_exit 1 "wrong value $value (!= 1) in $host:$resource_proc_sys_mars_reset_emergency_file" + fi +} + # compare actual data on data device with data written in # resource_dd_until_mars_dir_full function resource_check_data_on_data_device @@ -307,16 +335,26 @@ function resource_up return $rc } -function resource_rm_resource_dir +function resource_rm_resource_dir_all { local res=$1 host local res_dir=${resource_dir_list[$res]} + if [ -z "$res_dir" ];then + lib_exit 1 " to resource $res no resource dir found in resource_dir_list" + fi cluster_rmmod_mars_all for host in "${main_host_list[@]}"; do - lib_vmsg " removing $host:$res_dir/*" - lib_remote_idfile $host "rm -rf $res_dir/*" || lib_exit 1 + local mars_lv=${cluster_mars_dir_lv_name_list[$host]} + local mars_dev=$(lv_config_get_lv_device $mars_lv) + lib_vmsg " check whether mars device $host:$mars_dev exists" + if lib_remote_idfile $host "ls -l $mars_dev"; then + cluster_mount_mars_dir $host + lib_vmsg " removing $host:$res_dir" + lib_remote_idfile $host "rm -rf $res_dir" || lib_exit 1 + lib_remote_idfile $host "mkdir $res_dir" || lib_exit 1 + fi done } @@ -372,7 +410,7 @@ function resource_check_data_link lib_linktree_check_link $host "$link" $dev } -function resource_get_name_data_device +function resource_get_data_device { local res=$1 echo /dev/mars/$res @@ -381,14 +419,15 @@ function resource_get_name_data_device function resource_is_data_device_mounted { local host=$1 res=$2 rc - local dev=$(resource_get_name_data_device $res) - mount_is_device_mounted $host $dev + local dev=$(resource_get_data_device $res) + local mount_point + mount_is_device_mounted $host $dev "mount_point" } function resource_check_data_device_after_create { local host=$1 res=$2 - local dev=$(resource_get_name_data_device $res) + local dev=$(resource_get_data_device $res) local waited=0 rc while true; do lib_vmsg " checking existence of device $dev on $host" @@ -409,12 +448,13 @@ function resource_check_data_device_after_create function resource_check_mount_and_rmmod_possibilities { local host=$1 res=$2 - local data_dev=$(resource_get_name_data_device $res) + local data_dev=$(resource_get_data_device $res) + local mount_point resource_check_mount_point_directories $host - if ! mount_is_device_mounted $host $data_dev ${resource_mount_point_list[$res]} + if ! mount_is_device_mounted $host $data_dev "mount_point" then mount_mount $host $data_dev ${resource_mount_point_list[$res]} \ - ${resource_fs_type_list[$res]} + ${resource_fs_type_list[$res]} || lib_exit 1 fi resource_check_whether_rmmod_mars_fails $host $data_dev mount_umount $host $data_dev ${resource_mount_point_list[$res]} || \ diff --git a/test_suite/scripts/modules/45_multires.sh b/test_suite/scripts/modules/45_multires.sh new file mode 100644 index 00000000..ff88ceea --- /dev/null +++ b/test_suite/scripts/modules/45_multires.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +function multires_prepare +{ + cluster_umount_mars_dir_all + lv_config_prepare + lv_config_run + cluster_mount_mars_dir_all + resource_rm_resource_dir_all + cluster_create + cluster_join + cluster_insert_mars_module_all + multires_create_resources_all +} + +function multires_create_resources_all +{ + local primary_host=${main_host_list[0]} + local secondary_host=${main_host_list[1]} + local res lv_dev count=0 maxwait=20 + for res in ${lv_config_name_list[@]}; do + local lv_dev=$(lv_config_get_lv_device $res) + marsadm_do_cmd $primary_host "create-resource --force" "$res $lv_dev" + marsadm_do_cmd $primary_host "wait-resource" "$res is-device" + lib_remote_idfile $primary_host marsview $res + marsadm_do_cmd $secondary_host "join-resource --force" "$res $lv_dev" + lib_remote_idfile $secondary_host marsview $res + while true;do + if marsview_check $secondary_host $res "repl" "-S..-" \ + && marsview_check $secondary_host $res "disk" "Inconsistent" + then + marsadm_do_cmd $secondary_host "pause-sync" $res + marsadm_do_cmd $secondary_host "fake-sync" $res + marsadm_do_cmd $secondary_host "resume-sync" $res + continue + fi + if marsview_check $secondary_host $res "repl" "-SFA-" \ + && marsview_check $secondary_host $res "disk" "Uptodate" + then + break; + fi + sleep 1 + let count+=1 + if [ $count -ge $maxwait ]; then + lib_exit 1 "maxwait $maxwait exceeded" + fi + lib_vmsg " waited $count for secondary to become up to date" + if marsview_check $secondary_host $res "disk" "Detached"; then + lib_vmsg " try to join $res again" + marsadm_do_cmd $secondary_host "join-resource --force" $res \ + $lv_dev + fi + done + done +} diff --git a/test_suite/scripts/modules/55_remote_dev.sh b/test_suite/scripts/modules/55_remote_dev.sh index eb6d160e..3188d479 100644 --- a/test_suite/scripts/modules/55_remote_dev.sh +++ b/test_suite/scripts/modules/55_remote_dev.sh @@ -62,11 +62,11 @@ function remote_dev_run lib_rw_stop_writing_data_device $writer_script "write_count" - local lilo_label_name="${main_host_bootloader_label_list[$secondary_host]}" + local boot_label_name="${main_host_bootloader_label_list[$secondary_host]}" local mars_dev=$(lv_config_get_lv_device ${cluster_mars_dir_lv_name_list[$secondary_host]}) crash_reboot $secondary_host "" $mars_dev $crash_maxtime_reboot \ $crash_maxtime_to_become_unreachable \ - $lilo_label_name + $boot_label_name remote_dev_remove_magic_links $primary_host $secondary_host } @@ -75,7 +75,7 @@ function remote_dev_remove_magic_links local hosts="$@" host # to guarantee persistence of removal on all hosts - mount_umount_data_device + mount_umount_data_device_all cluster_rmmod_mars_all for host in $hosts; do diff --git a/test_suite/scripts/modules/58_resize.sh b/test_suite/scripts/modules/58_resize.sh index 3ddf2e2d..fa3e0093 100644 --- a/test_suite/scripts/modules/58_resize.sh +++ b/test_suite/scripts/modules/58_resize.sh @@ -97,7 +97,7 @@ function resize_do_resize [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local primary_host=$1 secondary_host=$2 res=$3 dev=$4 data_dev_size_new=$5 local mars_data_dev_size_new=$6 - local host time_waited + local host time_waited net_throughput for host in $primary_host $secondary_host; do lv_config_resize_device $host $dev $data_dev_size_new @@ -120,7 +120,8 @@ function resize_do_resize lib_wait_until_action_stops "syncstatus" $secondary_host $res \ $resize_maxtime_sync \ - $resize_time_constant_sync "time_waited" + $resize_time_constant_sync "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited" } @@ -132,6 +133,7 @@ function resize_resize_to_orig_size resource_leave_all for host in $primary_host $secondary_host; do lv_config_resize_device $host $dev $data_dev_size_orig + lib_remote_check_device_fs $host $dev ${resource_fs_type_list[$res]} done } diff --git a/test_suite/scripts/modules/65_switch2primary.sh b/test_suite/scripts/modules/65_switch2primary.sh index a2dc341b..d70e10ea 100644 --- a/test_suite/scripts/modules/65_switch2primary.sh +++ b/test_suite/scripts/modules/65_switch2primary.sh @@ -52,7 +52,7 @@ function switch2primary_run count=0 while true; do - mount_umount_data_device + mount_umount_data_device_all rc=$? if [ $rc -ne 0 ]; then let count+=1 diff --git a/test_suite/scripts/modules/68_crash.sh b/test_suite/scripts/modules/68_crash.sh index 9e90c644..6d3205d1 100644 --- a/test_suite/scripts/modules/68_crash.sh +++ b/test_suite/scripts/modules/68_crash.sh @@ -25,10 +25,11 @@ function crash_run local primary_host=${main_host_list[0]} local secondary_host=${main_host_list[1]} local mars_dev=$(lv_config_get_lv_device ${cluster_mars_dir_lv_name_list[$primary_host]}) - local lilo_label_name="${main_host_bootloader_label_list[$primary_host]}" + local boot_label_name="${main_host_bootloader_label_list[$primary_host]}" local res=${resource_name_list[0]} local dev=$(lv_config_get_lv_device $res) local writer_pid writer_script logfile length_logfile time_waited + local net_throughput local waited=0 error_ocurred=0 mount_mount_data_device @@ -46,7 +47,7 @@ function crash_run $crash_aio_sync_mode crash_reboot $primary_host $secondary_host $mars_dev $crash_maxtime_reboot \ $crash_maxtime_to_become_unreachable \ - $lilo_label_name + $boot_label_name lib_linktree_print_linktree $primary_host @@ -60,12 +61,14 @@ function crash_run $crash_maxtime_state_constant lib_wait_until_action_stops "syncstatus" $secondary_host $res \ $crash_maxtime_sync \ - $crash_time_constant_sync "time_waited" + $crash_time_constant_sync "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited" lib_wait_until_fetch_stops "crash" $secondary_host $primary_host $res \ - "logfile" "length_logfile" "time_waited" + "logfile" "length_logfile" "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited" @@ -91,13 +94,14 @@ function crash_run function crash_write_data_device_and_calculate_checksums { local primary_host=$1 secondary_host=$2 res=$3 dev=$4 - local writer_pid writer_script write_count time_waited + local writer_pid writer_script write_count time_waited net_throughput mount_mount_data_device lib_rw_start_writing_data_device "writer_pid" "writer_script" 0 0 $res lib_rw_stop_writing_data_device $writer_script "write_count" lib_wait_until_action_stops "replay" $secondary_host $res \ - $crash_maxtime_sync \ - $crash_time_constant_sync "time_waited" + $crash_maxtime_apply \ + $crash_time_constant_apply "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: apply time: $time_waited" @@ -113,7 +117,7 @@ function crash_reboot [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local primary_host=$1 secondary_host=$2 mars_dev=$3 maxtime_to_reboot=$4 local maxtime_to_become_unreachable=$5 - local lilo_label_name=$6 + local boot_label_name=$6 local pids_to_kill host if [ -z "$crash_print_linktree_during_reboot" ]; then @@ -123,8 +127,10 @@ function crash_reboot then lib_exit 1 "to print symlink trees secondary_host must be given" fi - install_mars_activate_kernel_to_boot_with_lilo $primary_host \ - $lilo_label_name + if [ "${main_host_bootloader_list[$primary_host]}" = "lilo" ]; then + install_mars_activate_kernel_to_boot_with_lilo $primary_host \ + $boot_label_name + fi main_error_recovery_functions["lib_rw_stop_scripts"]= diff --git a/test_suite/scripts/modules/70_file_destroy.sh b/test_suite/scripts/modules/70_file_destroy.sh index 6a9dc57f..f9aaa4ae 100644 --- a/test_suite/scripts/modules/70_file_destroy.sh +++ b/test_suite/scripts/modules/70_file_destroy.sh @@ -26,7 +26,7 @@ function file_destroy_run local secondary_host="${main_host_list[1]}" local res=${resource_name_list[0]} local logfile length_logfile writer_pid writer_script write_count - local time_waited + local time_waited net_throughput lib_wait_for_initial_end_of_sync $secondary_host $res \ $resource_maxtime_initial_sync \ @@ -48,7 +48,8 @@ function file_destroy_run lib_rw_stop_writing_data_device $writer_script "write_count" lib_wait_until_fetch_stops "file_destroy" $secondary_host $primary_host \ - $res "logfile" "length_logfile" "time_waited" + $res "logfile" "length_logfile" "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited" @@ -62,7 +63,7 @@ function file_destroy_run lib_wait_until_action_stops "replay" $secondary_host $res \ $file_destroy_maxtime_apply \ $file_destroy_time_constant_apply \ - "time_waited" + "time_waited" 0 "net_throughput" lib_vmsg " ${FUNCNAME[0]}: apply time: $time_waited" marsview_check $secondary_host $res "disk" "Outdated\[.*A.*\]" || lib_exit 1 diff --git a/test_suite/scripts/modules/71_lib_wait.sh b/test_suite/scripts/modules/71_lib_wait.sh index 21d9f104..6a2b58d4 100644 --- a/test_suite/scripts/modules/71_lib_wait.sh +++ b/test_suite/scripts/modules/71_lib_wait.sh @@ -17,12 +17,58 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -function lib_wait_until_fetch_stops +function lib_wait_until_logfile_has_length { [ $# -eq 7 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 logfile=$2 length_logfile=$3 + local varname_time_waited=$4 maxwait=$5 check_net_throughput=$6 varname_net_throughput=$7 + local act_length + local waited=0 start_time=$(date +'%s') end_time + local my_net_throughput=0 net_throughput_sum=0 net_check_count=0 + + lib_vmsg " waiting for $host:$logfile to grow to $length_logfile" + + while true; do + lib_vmsg " get length of $host:$logfile" + act_length=$(file_handling_get_file_length $secondary_host $logfile) \ + || lib_exit 1 + if [ $act_length -ge $length_logfile ]; then + end_time=$(date +'%s') + break + fi + sleep 1 + let waited+=1 + lib_vmsg " waited $waited for $logfile act = $act_length, req = $length_logfile" + if [ $waited -eq $maxwait ]; then + lib_exit 1 "maxwait $maxwait exceeded" + fi + if [ $check_net_throughput -eq 1 ]; then + if [ $(( $waited % $perftest_check_net_throughput_intervall )) -eq 0 ]; then + perftest_check_tcp_connection $primary_host $secondary_host \ + "my_net_throughput" + let net_throughput_sum+=$my_net_throughput + let net_check_count+=1 + fi + fi + done + eval $varname_time_waited=$(($end_time - $start_time)) + if [ $check_net_throughput -eq 1 ]; then + local rate + if [ $net_check_count -eq 0 ]; then + rate=0 + else + rate=$(($net_throughput_sum / $net_check_count)) + fi + eval $varname_net_throughput=$rate + fi +} + +function lib_wait_until_fetch_stops +{ + [ $# -eq 9 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local module=$1 secondary_host=$2 primary_host=$3 res=$4 local varname_logfile=$5 varname_length_logfile=$6 - local varname_time_waited=$7 + local varname_time_waited=$7 check_net_throughput=$8 varname_net_throughput=$9 local maxtime_fetch time_constant_fetch var v for var in maxtime_fetch time_constant_fetch; do @@ -37,17 +83,20 @@ function lib_wait_until_fetch_stops $maxtime_fetch \ $time_constant_fetch \ $varname_logfile $varname_length_logfile \ - $varname_time_waited + $varname_time_waited \ + $check_net_throughput $varname_net_throughput } function lib_wait_internal_until_fetch_stops { - [ $# -eq 8 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + [ $# -eq 10 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local secondary_host=$1 res=$2 primary_host=$3 maxwait=$4 inactive_wait=$5 local varname_logfile=$6 varname_logfile_length=$7 varname_time_waited=$8 + local check_net_throughput=$9 varname_net_throughput=${10} local inactive_waited=0 msg local my_logfile length file_and_length file_and_length_old="x" local waited=0 msg start_time=$(date +'%s') end_time + local my_net_throughput=0 net_throughput_sum=0 net_check_count=0 while [ $waited -lt $maxwait ]; do my_logfile=$(marsadm_get_last_logfile $secondary_host $res \ $primary_host) || lib_exit 1 @@ -70,11 +119,29 @@ function lib_wait_internal_until_fetch_stops let waited+=1 msg=" waited $waited for $my_logfile act = $file_and_length, old = $file_and_length_old" lib_vmsg "$msg" + if [ $check_net_throughput -eq 1 ]; then + if [ $(( $waited % $perftest_check_net_throughput_intervall )) -eq 0 ]; then + perftest_check_tcp_connection $primary_host $secondary_host \ + "my_net_throughput" + let net_throughput_sum+=$my_net_throughput + let net_check_count+=1 + fi + fi + file_and_length_old="$file_and_length" done if [ $waited -eq $maxwait ]; then lib_exit 1 "$msg" fi + if [ $check_net_throughput -eq 1 ]; then + local rate + if [ $net_check_count -eq 0 ]; then + rate=0 + else + rate=$(($net_throughput_sum / $net_check_count)) + fi + eval $varname_net_throughput=$rate + fi eval $varname_logfile_length=$length eval $varname_logfile=$my_logfile eval $varname_time_waited=$(($end_time - $start_time)) @@ -85,12 +152,14 @@ function lib_wait_internal_until_fetch_stops # the time waited is returned in the variable, whichs name is given by $6 function lib_wait_until_action_stops { - [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + [ $# -eq 8 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local action=$1 host=$2 res=$3 maxwait=$4 inactive_wait=$5 - local varname_time_waited=$6 + local varname_time_waited=$6 check_net_throughput=$7 + local varname_net_throughput=$8 local waited=0 link_value link_value_old="x" local inactive_waited=0 msg start_time=$(date +'%s') end_time local link=$(lib_linktree_get_res_host_linkname $host $res $action) + local my_net_throughput=0 net_throughput_sum=0 net_check_count=0 while [ $waited -lt $maxwait ]; do link_value=$(lib_remote_idfile $host readlink $link) || \ lib_exit 1 "cannot read link $link" @@ -109,12 +178,29 @@ function lib_wait_until_action_stops let waited+=1 msg=" waited $waited for $action to stop. old = $link_value_old, act = $link_value" lib_vmsg "$msg" + if [ $check_net_throughput -eq 1 ]; then + if [ $(( $waited % $check_net_throughput )) -eq 0 ]; then + perftest_check_tcp_connection $primary_host $secondary_host \ + "my_net_throughput" + let net_throughput_sum+=$my_net_throughput + let net_check_count+=1 + fi + fi link_value_old="$link_value" done if [ $waited -eq $maxwait ]; then lib_exit 1 "$msg" fi eval $varname_time_waited=$(($end_time - $start_time)) + if [ $check_net_throughput -eq 1 ]; then + local rate + if [ $net_check_count -eq 0 ]; then + rate=0 + else + rate=$(($net_throughput_sum / $net_check_count)) + fi + eval $varname_net_throughput=$rate + fi } function lib_wait_for_initial_end_of_sync @@ -122,8 +208,10 @@ function lib_wait_for_initial_end_of_sync [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local secondary_host=$1 res=$2 maxwait=$3 inactive_wait=$4 local varname_time_waited=$5 + local net_throughput lib_wait_until_action_stops "syncstatus" $secondary_host $res $maxwait \ - $inactive_wait $varname_time_waited + $inactive_wait $varname_time_waited 0 \ + "net_throughput" # after sync disk state must be Outdated || Uptodate marsview_check $secondary_host $res "disk" ".*date.*" || lib_exit 1 } @@ -134,6 +222,7 @@ function lib_wait_for_secondary_to_become_uptodate local module_name=$1 secondary_host=$2 primary_host=$3 res=$4 local dev=$5 dev_size_to_compare=$6 local host role logfile length_logfile time_waited write_count + local net_throughput mount_point local maxtime_apply time_constant_apply str var for str in "maxtime" "time_constant"; do @@ -145,15 +234,17 @@ function lib_wait_for_secondary_to_become_uptodate done lib_wait_until_fetch_stops $module_name $secondary_host $primary_host $res \ - "logfile" "length_logfile" "time_waited" + "logfile" "length_logfile" "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]} called from ${FUNCNAME[1]}: fetch time: $time_waited" file_handling_check_equality_of_file_lengths $logfile $primary_host \ $secondary_host $length_logfile lib_wait_until_action_stops "replay" $secondary_host $res \ - $maxtime_apply \ - $time_constant_apply "time_waited" + $maxtime_apply \ + $time_constant_apply "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]} called from ${FUNCNAME[1]}: apply time: $time_waited" @@ -163,10 +254,38 @@ function lib_wait_for_secondary_to_become_uptodate marsview_check $host $res "repl" "-SFA-" || lib_exit 1 done - if mount_is_device_mounted $primary_host $dev; then - mount_umount $primary_host $dev ${resource_mount_point_list[$res]} + if mount_is_device_mounted $primary_host $dev "mount_point"; then + mount_umount $primary_host $dev $mount_point fi lib_rw_compare_checksums $primary_host $secondary_host $dev \ $dev_size_to_compare "" "" } + +function lib_wait_until_apply_has_reached_length +{ + [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local secondary_host=$1 res=$2 logfile=$3 req_applied_length=$4 maxwait=$5 + local link=$(lib_linktree_get_res_host_linkname $secondary_host $res "replay") + local link_value waited=0 act_applied_length + while true; do + lib_vmsg " get applied length of $secondary_host:$logfile" + link_value=$(lib_remote_idfile $secondary_host readlink $link) \ + || lib_exit 1 "cannot read link $link" + link_value=(${link_value//,/ }) + act_applied_length=${link_value[1]} + if ! expr "$act_applied_length" : '^[0-9][0-9]*$' >/dev/null; then + lib_exit 1 "cannot determine applied length from link_value ${link[@]}" + fi + if [ $act_applied_length -ge $req_applied_length ]; then + break + fi + sleep 1 + let waited+=1 + lib_vmsg " waited $waited for apply of $logfile act = $act_applied_length, req = $req_applied_length" + if [ $waited -eq $maxwait ]; then + lib_exit 1 "maxwait $maxwait exceeded" + fi + done +} + diff --git a/test_suite/scripts/modules/72_perftest.sh b/test_suite/scripts/modules/72_perftest.sh new file mode 100644 index 00000000..00f0513f --- /dev/null +++ b/test_suite/scripts/modules/72_perftest.sh @@ -0,0 +1,743 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +function perftest_check_variables +{ + [ -z "$perftest_action" ] && lib_exit 1 "no action defined" + case $perftest_action in # (( + apply|fetch|write|sync|fetch_and_apply) : + ;; + *) lib_exit 1 "invalid action $perftest_action" + ;; + esac +} + +function perftest_run +{ + local primary_host=${main_host_list[0]} + local secondary_host=${main_host_list[1]} + local res=${resource_name_list[0]} + + perftest_check_variables + + perftest_prepare_${perftest_action} $primary_host $secondary_host $res \ + $perftest_parallel_writer \ + $perftest_result_type \ + ${#resource_name_list[*]} + + case $perftest_action in # (((( + fetch|sync|fetch_and_apply) perftest_start_and_check_nttcp $primary_host $secondary_host + ;; + *) : + ;; + esac + + perftest_do_${perftest_action} $primary_host $secondary_host $res \ + $perftest_parallel_writer \ + $perftest_result_type \ + ${#resource_name_list[*]} + perftest_finish $secondary_host +} + +function perftest_finish +{ + local primary_host=${main_host_list[0]} + local secondary_host=${main_host_list[1]} + perftest_stop_nttcp $secondary_host +} + +function perftest_stop_nttcp +{ + local secondary_host=$1 + for i in "kill" "check"; do + if lib_remote_idfile $secondary_host \ + 'pgrep -f "'"$perftest_nttcp_start_cmd"'"' + then + if [ "$i" = "kill" ]; then + lib_vmsg " trying to kill nttcp on $secondary_host" + lib_remote_idfile $secondary_host \ + 'kill -1 $(pgrep -f "'"$perftest_nttcp_start_cmd"'")' + else + lib_vmsg " could not kill nttcp on $secondary_host" + break + fi + else + if [ "$i" = "kill" ]; then + lib_vmsg " no process $perftest_nttcp_start_cmd running" + break + else + lib_vmsg " killed process $perftest_nttcp_start_cmd" + fi + fi + sleep 1 + done +} + +function perftest_start_and_check_nttcp +{ + local primary_host=$1 secondary_host=$2 + local net_throughput + local cmd="nttcp -p $perftest_nttcp_port -r -i &" + local host i + if [ ${perftest_check_net_throughput:-0} -eq 0 ]; then + lib_exit 1 "perftest_check_net_throughput not set or 0" + fi + for host in $primary_host $secondary_host; do + lib_vmsg " checking whether nttcp is installed on $host" + lib_remote_idfile $host "type nttcp" || \ + lib_exit 1 "nttcp not installed on $host" + done + for i in "start" "check"; do + if ! lib_remote_idfile $secondary_host \ + "pgrep -f '"$perftest_nttcp_start_cmd"'" + then + if [ "$i" = "start" ]; then + lib_vmsg " starting nttcp receiver on $secondary_host" + lib_remote_idfile $secondary_host "$perftest_nttcp_start_cmd" & + else + lib_vmsg " could not start receiver on $secondary_host" + return + fi + else + if [ "$i" = "start" ]; then + lib_vmsg " nttcp $perftest_nttcp_start_cmd already running on $secondary_host" + break + else + lib_vmsg " nttcp $perftest_nttcp_start_cmd started on $secondary_host" + fi + fi + sleep 1 + done + perftest_check_tcp_connection $primary_host $secondary_host "net_throughput" +} + +function perftest_check_tcp_connection +{ + [ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local primary_host=$1 secondary_host=$2 varname_throughput=$3 + local cmd="nttcp -n16384 -f %9b%8.2rt%8.2ct%15.4rbr%15.4cbr%8c%10.2rcr%10.2ccr -p $perftest_nttcp_port -T $secondary_host" + local nttcp_out mbit_per_second + if [ ${perftest_check_net_throughput:-0} -eq 0 ]; then + lib_exit 1 "perftest_check_net_throughput not set or 0" + fi + lib_vmsg " checking tcp via nttcp on $primary_host" + nttcp_out=($(lib_remote_idfile $primary_host "$cmd")) + echo "${nttcp_out[*]}" + # nttcp_out looks like + # Bytes Real s CPU s Real-MBit/s CPU-MBit/s Calls Real-C/s CPU-C/s + # l 67108864 0.55 0.03 971.1935 17895.6971 16384 29638.47 546133.33 + # 1 67108864 0.62 0.09 862.4709 5965.2324 20383 32744.83 226477.78 + # and we need the 971 Mbit/s + mbit_per_second=${nttcp_out[14]} + eval $varname_throughput=${mbit_per_second%.*} +} + + + +function perftest_remove_data_file +{ + local data_file=$1 + shift + local hosts="$@" host + for host in $hosts; do + lib_vmsg " removing $host:$data_file" + lib_remote_idfile $host "rm -f $data_file" || lib_exit 1 + done +} + +function perftest_do_write +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local writer_pid writer_script write_count writer_start + local writer_rate + writer_start=$(date +'%s') + lib_rw_start_writing_data_device "writer_pid" "writer_script" 0 0 $res + lib_vmsg " sleep $perftest_write_time" + sleep $perftest_write_time + lib_rw_stop_writing_data_device $writer_script "write_count" + writer_rate=$(perftest_get_rate_per_minute $writer_start $(date +'%s') \ + $write_count) + main_error_recovery_functions["lib_rw_stop_scripts"]= + mount_umount_data_device_all + lib_vmsg " ${FUNCNAME[0]}: do_write rate: $writer_rate" + perftest_check_result $writer_rate $primary_host "write" $parallel_writer \ + $result_type $no_resources \ + $(perftest_get_write_subcase_id) -1 +} + +function perftest_get_rate_per_minute +{ + [ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local start=$1 end=$2 count=$3 + echo $(( (60 * $count ) / ($end - $start) )) +} + +function perftest_via_rsync +{ + local secondary_host=$1 primary_host=$2 data_file=$3 + local varname_time_waited=$4 + local start=$(date +'%s') time_waited + + perftest_do_rsync $secondary_host $primary_host $data_file + + time_waited=$(($(date +'%s') - $start)) + eval $varname_time_waited=$time_waited +} + +function perftest_do_rsync +{ + local secondary_host=$1 primary_host=$2 data_file=$3 + lib_vmsg " syncing $primary_host:$data_file to $secondary_host" + lib_remote_idfile $secondary_host \ + "rsync -av -e ssh root@$primary_host:$data_file $data_file" || \ + lib_exit 1 +} + +function perftest_start_parallel_writer +{ + [ $# -eq 4 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local varname_writer_start=$1 varname_writer_pid=$2 varname_writer_script=$3 + local res=$4 + mount_mount_data_device + eval $varname_writer_start=$(date +'%s') + lib_rw_start_writing_data_device $varname_writer_pid \ + $varname_writer_script 0 0 $res +} + +function perftest_finish_parallel_writer +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 writer_script=$2 writer_start=$3 + local action=$4 no_resources=$5 subcase_id="$6" + local write_count writer_rate + local caller="${BASH_SOURCE[1]}:${FUNCNAME[1]}:${BASH_LINENO[0]}" + + lib_rw_stop_writing_data_device $writer_script "write_count" + writer_rate=$(perftest_get_rate_per_minute $writer_start $(date +'%s') \ + $write_count) + main_error_recovery_functions["lib_rw_stop_scripts"]= + lib_vmsg " $caller: do_write rate: $writer_rate" + perftest_check_result $writer_rate $host write_while_$action 0 \ + "loops_per_min" $no_resources "$subcase_id" -1 + + mount_umount_data_device_all +} + +function perftest_via_mars_sync +{ + [ $# -eq 8 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local primary_host=$1 secondary_host=$2 res=$3 dev=$4 + local parallel_writer=$5 result_type=$6 no_resources=$7 subcase_id=$8 + local synctime net_throughput + local start_sync writer_pid writer_script writer_start + + marsadm_do_cmd $secondary_host "invalidate" $res || lib_exit 1 + + + if [ $parallel_writer -eq 1 ]; then + writer_start=$(date +'%s') + perftest_start_parallel_writer "writer_start" "writer_pid" \ + "writer_script" $res + fi + + marsadm_do_cmd $secondary_host "up" $res || lib_exit 1 + + lib_wait_until_action_stops "syncstatus" $secondary_host $res \ + $perftest_maxtime_sync \ + $perftest_time_constant_sync \ + "synctime" 1 "net_throughput" + + perftest_check_result $synctime $secondary_host $perftest_action \ + $parallel_writer "time" $no_resources "$subcase_id" $net_throughput + + if [ $parallel_writer -eq 1 ]; then + perftest_finish_parallel_writer $primary_host $writer_script \ + $writer_start $perftest_action \ + $no_resources "$subcase_id" + lib_vmsg " recreating all resources" + resource_recreate_all + + else + lib_rw_compare_checksums $primary_host $secondary_host $dev 0 "" "" + fi +} + +function perftest_generate_data_file +{ + [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local primary_host=$1 secondary_host=$2 dev=$3 dev_size=$4 + local data_file=$5 file_size_in_kb=$(($dev_size * 1024 * 1024)) + local host + + lib_vmsg " generating file $primary_host:$data_file ($file_size_in_kb KB) from $dev" + lib_remote_idfile $primary_host \ + "dd if=$dev of=$data_file bs=1024 count=$file_size_in_kb" || \ + lib_exit 1 + perftest_do_rsync $secondary_host $primary_host $data_file + + for host in $primary_host $secondary_host; do + lib_remote_idfile $host "ls -l --full-time $data_file" || lib_exit 1 + done +} + +function perftest_sysctrl_sync_modus +{ + local hosts="$@" host + local mars_fast_sync_mode=0 + if [ "$perftest_sync_mode" = "fast_sync" ]; then + mars_fast_sync_mode=1 + fi + for host in $hosts; do + lib_vmsg " setting fast sync mode to $mars_fast_sync_mode on $host" + lib_remote_idfile $host \ + "echo $mars_fast_sync_mode > $perftest_sync_mode_proc_file" \ + || lib_exit 1 + done +} + +function perftest_patch_data_device +{ + [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 dev=$2 dev_size_in_kb=$(($3 * 1024 * 1024)) + local patch_length_in_kb=$4 no_of_patches=$5 + local offset=0 bs=1024 remaining=$dev_size_in_kb + + while [ $offset -lt $((dev_size_in_kb - $patch_length_in_kb)) ]; do + lib_vmsg " patching $dev at $offset KB with $patch_length_in_kb KB" + lib_remote_idfile $host \ + "yes :$offset: | dd of=$dev bs=$bs skip=$offset count=$patch_length_in_kb" || lib_exit 1 + offset=$(($offset + ($dev_size_in_kb / $no_of_patches))) + done +} + +function perftest_prepare_sync +{ + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local dev=$(lv_config_get_lv_device $res) + local dev_size=$(lv_config_get_lv_size $res) + local time_waited + + lib_wait_for_initial_end_of_sync $secondary_host $res \ + $resource_maxtime_initial_sync \ + $resource_time_constant_initial_sync \ + "time_waited" + lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited" + + + if [ "$perftest_sync_mode" != "rsync" ]; then + perftest_sysctrl_sync_modus $primary_host $secondary_host + else + perftest_generate_data_file $primary_host $secondary_host $dev \ + $dev_size $perftest_data_file + fi +} + +function perftest_prepare_apply +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local data_dev=$(resource_get_data_device $res) + local logfile length_logfile time_waited net_throughput + + perftest_check_and_get_required_result $secondary_host "apply" \ + $parallel_writer $result_type \ + $no_resources $perftest_logfile_size_in_gb >/dev/null \ + || lib_exit 1 + + perftest_prepare_resource $res $secondary_host + + marsadm_do_cmd $secondary_host "pause-replay" $res || lib_exit 1 + + perftest_write_to_device $primary_host $res $data_dev $perftest_data_in_gb_to_write + + lib_wait_until_fetch_stops "perftest" $secondary_host $primary_host $res \ + "logfile" "length_logfile" "time_waited" 0 \ + "net_throughput" + lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited" + if [ $(lib_round_to_gb $length_logfile) -ne $$perftest_logfile_size_in_gb ] + then + lib_exit 1 "req. logfile length = $$perftest_logfile_size_in_gb != $(lib_round_to_gb $length_logfile) = act. logfile length" + fi + marsadm_do_cmd $secondary_host "disconnect" $res || lib_exit 1 + +} + +function perftest_do_apply +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local time_waited net_throughput + local writer_start writer_pid writer_script + + if [ $parallel_writer -eq 1 ]; then + writer_start=$(date +'%s') + perftest_start_parallel_writer "writer_start" "writer_pid" \ + "writer_script" $res + fi + + marsadm_do_cmd $secondary_host "resume-replay" $res || lib_exit 1 + + lib_wait_until_action_stops "replay" $secondary_host $res \ + $perftest_maxtime_apply \ + $perftest_time_constant_apply "time_waited" \ + 0 "net_throughput" + lib_vmsg " ${FUNCNAME[0]}: do_$perftest_action time: $time_waited" + + perftest_check_result $time_waited $secondary_host $perftest_action \ + $parallel_writer $result_type $no_resources \ + $perftest_logfile_size_in_gb -1 + if [ $parallel_writer -eq 1 ]; then + perftest_finish_parallel_writer $primary_host $writer_script \ + $writer_start $perftest_action \ + $no_resources $perftest_logfile_size_in_gb + fi +} + +function perftest_prepare_resource +{ + local res=$1 secondary_host=$2 + resource_rm_resource_dir_all $res + resource_run + marsview_wait_for_state $secondary_host $res "disk" "Uptodate" \ + $perftest_maxtime_state_constant || lib_exit 1 +} + +function perftest_prepare_fetch +{ + perftest_prepare_fetch_or_fetch_and_apply "$@" +} + +function perftest_prepare_fetch_and_apply +{ + perftest_prepare_fetch_or_fetch_and_apply "$@" +} + +function perftest_do_fetch +{ + perftest_do_fetch_or_fetch_and_apply "$@" +} + +function perftest_do_fetch_and_apply +{ + perftest_do_fetch_or_fetch_and_apply "$@" +} + +function perftest_prepare_fetch_or_fetch_and_apply +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local data_dev=$(resource_get_data_device $res) + local logfile logfile length_logfile + + perftest_check_and_get_required_result $secondary_host $perftest_action \ + $parallel_writer $result_type \ + $no_resources $perftest_logfile_size_in_gb >/dev/null \ + || lib_exit 1 + + perftest_prepare_resource $res $secondary_host + + marsadm_do_cmd $secondary_host "pause-replay" $res || lib_exit 1 + marsadm_do_cmd $secondary_host "disconnect" $res || lib_exit 1 + + perftest_write_to_device $primary_host $res $data_dev $perftest_data_in_gb_to_write + + perftest_check_length_last_logfile $primary_host $res $primary_host \ + $perftest_logfile_size_in_gb + +} + +function perftest_get_length_last_logfile +{ + local host=$1 res=$2 primary_host=$3 + local length_logfile + logfile=$(marsadm_get_last_logfile $host $res $primary_host) || lib_exit 1 + length_logfile=$(file_handling_get_file_length $host $logfile) || lib_exit 1 + echo $length_logfile +} + +function perftest_check_length_last_logfile +{ + [ $# -eq 4 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 res=$2 primary_host=$3 logfile_size_in_gb_req=$4 + local length_logfile + + length_logfile=$(perftest_get_length_last_logfile $host $res $primary_host) + + if [ $(lib_round_to_gb $length_logfile) -ne $logfile_size_in_gb_req ]; then + lib_exit 1 "req. logfile length = $logfile_size_in_gb_req != $(lib_round_to_gb $length_logfile) = act. logfile length" + fi +} + +function perftest_do_fetch_or_fetch_and_apply +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local logfile length_logfile time_waited net_throughput + local last_logfile_primary last_logfile_length_primary + local writer_start writer_pid writer_script + + # with parallel writing, fetch will never stop. Though we take the time + # until the actually last logfile has been fetched + if [ $parallel_writer -eq 1 ]; then + last_logfile_primary=$(marsadm_get_last_logfile $primary_host $res \ + $primary_host) || lib_exit 1 + last_logfile_length_primary=$(file_handling_get_file_length \ + $primary_host $last_logfile_primary) \ + || lib_exit 1 + lib_vmsg " last logfile $primary_host:$last_logfile_primary has length $last_logfile_length_primary" + + writer_start=$(date +'%s') + perftest_start_parallel_writer "writer_start" "writer_pid" \ + "writer_script" $res + fi + + marsadm_do_cmd $secondary_host "connect" $res || lib_exit 1 + if [ $perftest_action = "fetch_and_apply" ]; then + marsadm_do_cmd $secondary_host "resume-replay" $res || lib_exit 1 + fi + + if [ $parallel_writer -eq 0 ]; then + lib_wait_until_fetch_stops "perftest" $secondary_host $primary_host \ + $res "logfile" "length_logfile" \ + "time_waited" 1 "net_throughput" + + file_handling_check_equality_of_file_lengths $logfile $primary_host \ + $secondary_host \ + $length_logfile + else + lib_wait_until_logfile_has_length $secondary_host \ + $last_logfile_primary \ + $last_logfile_length_primary \ + "time_waited" \ + $perftest_maxtime_fetch 1 \ + "net_throughput" + if [ $perftest_action = "fetch_and_apply" ]; then + lib_wait_until_apply_has_reached_length $secondary_host $res $last_logfile_primary \ + $last_logfile_length_primary \ + $perftest_wait_for_apply_to_stop_after_fetch_end + fi + + fi + + lib_vmsg " ${FUNCNAME[0]}: do_fetch time: $time_waited" + + perftest_check_result $time_waited $secondary_host $perftest_action \ + $parallel_writer $result_type $no_resources \ + $perftest_logfile_size_in_gb $net_throughput + + if [ $parallel_writer -eq 1 ]; then + perftest_finish_parallel_writer $primary_host $writer_script \ + $writer_start $perftest_action \ + $no_resources $perftest_logfile_size_in_gb + fi +} + +function perftest_get_result_index +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 action=$2 parallel_writer=$3 result_type=$4 no_resources=$5 + local subcase_id="$6" + echo "$host,$action,$parallel_writer,$result_type,$no_resources,$subcase_id" +} + +function perftest_check_and_get_required_result +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 action=$2 parallel_writer=$3 result_type=$4 no_resources=$5 + local subcase_id="$6" + local result_index="$(perftest_get_result_index $host $action \ + $parallel_writer $result_type \ + $no_resources $subcase_id)" + if [ -z "${perftest_required_result_list[$result_index]}" ]; then + lib_exit 1 "no value in perftest_required_result_list for index $result_index" + fi + echo ${perftest_required_result_list[$result_index]} +} + +function perftest_check_result +{ + [ $# -eq 8 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local act_result=$1 + local host=$2 action=$3 parallel_writer=$4 result_type=$5 no_resources=$6 + local subcase_id="$7" net_throughput=$8 + local req_result req_result_string act_diff_percentage max_diff_percentage + req_result_string=$(perftest_check_and_get_required_result $host $action \ + $parallel_writer $result_type \ + $no_resources $subcase_id) \ + || lib_exit 1 + req_result=${req_result_string%,*} + max_diff_percentage=${req_result_string#*,} + act_diff_percentage=$(( (($act_result - $req_result) * 100 ) / $req_result )) + lib_vmsg " checking result $act_result for index $(perftest_get_result_index \ + $host $action $parallel_writer \ + $result_type \ + $no_resources \ + $subcase_id), req. = $req_result max_diff = $max_diff_percentage%, act. diff percentage = $act_diff_percentage%, net rate = $net_throughput" + if [ $act_diff_percentage -lt -$max_diff_percentage \ + -o $act_diff_percentage -gt $max_diff_percentage ] + then + echo "$perftest_errortag_result_out_of_bounds: act. result $act_result differs more than $max_diff_percentage% from $req_result" >&2 + fi +} + +function perftest_write_to_device +{ + local host=$1 res=$2 dev=$3 data_in_gb_to_write=$4 + local bs=1024 dd_count=$((1024*1024)) + + lib_vmsg " writing $data_in_gb_to_write GB to $host:$dev" + for i in $(seq 1 1 $data_in_gb_to_write); do + lib_remote_idfile $host \ + 'yes $(printf "%0.1024d" '$i') | dd of='"$dev"' bs='"$bs"' count='$dd_count'' \ + || lib_exit 1 + done +} + +function perftest_get_sync_subcase_id +{ + local no_of_patches=$1 patch_length_in_kb=$2 sync_mode=$3 + echo "$no_of_patches:$patch_length_in_kb:$sync_mode" +} + +function perftest_do_sync +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local dev=$(lv_config_get_lv_device $res) + local dev_size=$(lv_config_get_lv_size $res) + local i patch_length_in_kb no_of_patches + local dev_to_patch + + if [ "$perftest_sync_mode" != "rsync" ]; then + dev_to_patch=$dev + else + dev_to_patch=$perftest_data_file + fi + + for i in ${!perftest_patch_length_list[@]}; do + no_of_patches=${perftest_number_of_patches_list[$i]} + patch_length_in_kb=${perftest_patch_length_list[$i]} + local subcase_id="$(perftest_get_sync_subcase_id $no_of_patches \ + $patch_length_in_kb \ + $perftest_sync_mode)" + perftest_check_and_get_required_result $secondary_host "sync" \ + $parallel_writer $result_type \ + $no_resources "$subcase_id" >/dev/null \ + || lib_exit 1 + lib_vmsg " patchlength $patch_length_in_kb, no of patches $no_of_patches" + if [ "$perftest_sync_mode" != "rsync" ]; then + marsadm_do_cmd $secondary_host "down" $res || lib_exit 1 + fi + + perftest_patch_data_device $secondary_host $dev_to_patch $dev_size \ + $patch_length_in_kb $no_of_patches + + if [ "$perftest_sync_mode" != "rsync" ]; then + + perftest_via_mars_sync $primary_host $secondary_host $res $dev \ + $parallel_writer $result_type $no_resources \ + "$subcase_id" + else + perftest_via_rsync $secondary_host $primary_host \ + $perftest_data_file "synctime" "$subcase_id" + fi + + done + + if [ "$perftest_sync_mode" == "rsync" ]; then + perftest_remove_data_file $perftest_data_file $primary_host \ + $secondary_host + fi +} + +function perftest_get_write_subcase_id +{ + local subcase_id + subcase_id="$perftest_write_time:$(lv_config_get_lv_size ${resource_name_list[0]}):$lib_rw_part_of_device_size_written_per_loop" + if [ $perftest_division_mars_device_data_device -eq 1 ]; then + subcase_id+=":$perftest_device_division" + fi + echo "$subcase_id" +} + +function perftest_prepare_write +{ + [ $# -eq 6 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + lib_vmsg " executing ${FUNCNAME[0]}" + local primary_host=$1 secondary_host=$2 res=$3 + local parallel_writer=$4 result_type=$5 no_resources=$6 + local data_dev=$(resource_get_data_device $res) + local logfile length_logfile time_waited + + if [ -z "$perftest_write_time" ]; then + lib_exit 1 "perftest_write_time not set" + fi + + perftest_check_and_get_required_result $primary_host "write" \ + $parallel_writer $result_type \ + $no_resources \ + $(perftest_get_write_subcase_id) >/dev/null \ + || lib_exit 1 + + perftest_prepare_resource $res $secondary_host + mount_mount_data_device + if [ $perftest_division_mars_device_data_device -eq 1 ]; then + perftest_switch_bbu_cache $primary_host + fi +} + +function perftest_switch_bbu_cache +{ + local host=$1 cmd + case $perftest_device_division in # ((( + separated_and_mars_dev_without_bbu_cache) + lib_vmsg " disabling bbu cache on $host" + for cmd in "${perftest_bbu_disable_cmd_list[@]}"; do + lib_remote_idfile $host $cmd || lib_exit 1 + done + ;; + separated_and_mars_dev_with_bbu_cache) + lib_vmsg " enabling bbu cache on $host" + for cmd in "${perftest_bbu_enable_cmd_list[@]}"; do + lib_remote_idfile $host $cmd || lib_exit 1 + done + ;; + same_controller) : + ;; + *) lib_exit 1 "invalid value $perftest_device_division for perftest_device_division in" + ;; + esac + for cmd in "${perftest_bbu_show_cmd_list[@]}"; do + lib_remote_idfile $host $cmd || lib_exit 1 + done +} diff --git a/test_suite/scripts/modules/72_synctest.sh b/test_suite/scripts/modules/72_synctest.sh deleted file mode 100644 index 4e3b808b..00000000 --- a/test_suite/scripts/modules/72_synctest.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/bin/bash -# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG -# -# Email: frank.liepold@1und1.de -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -function synctest_check_variables -{ - if [ ${#synctest_patch_length_list[@]} \ - -ne ${#synctest_number_of_patches_list[@]} ] - then - lib_exit 1 " different array lengths ${#synctest_patch_length_list[@]} != ${#synctest_number_of_patches_list[@]}" - fi -} - -function synctest_run -{ - local primary_host=${main_host_list[0]} - local secondary_host=${main_host_list[1]} - local res=${resource_name_list[0]} - local dev=$(lv_config_get_lv_device $res) - local dev_size=$(lv_config_get_lv_size $res) - local synctime i patch_length_kb no_of_patches - local synctimes dev_to_patch time_waited - - synctest_check_variables - - lib_wait_for_initial_end_of_sync $secondary_host $res \ - $resource_maxtime_initial_sync \ - $resource_time_constant_initial_sync \ - "time_waited" - lib_vmsg " ${FUNCNAME[0]}: sync time: $time_waited" - - - if [ $synctest_use_mars_sync -eq 1 ]; then - dev_to_patch=$dev - synctest_set_sync_modus $primary_host $secondary_host - else - dev_to_patch=$synctest_data_file - synctest_generate_data_file $primary_host $secondary_host $dev \ - $dev_size $synctest_data_file - fi - - if [ $synctest_parallel_writer -eq 1 ]; then - mount_mount_data_device - synctest_determine_write_rate $primary_host - fi - - - for i in ${!synctest_patch_length_list[@]}; do - patch_length_in_kb=${synctest_patch_length_list[$i]} - no_of_patches=${synctest_number_of_patches_list[$i]} - lib_vmsg " patchlength $patch_length_in_kb, no of patches $no_of_patches" - if [ $synctest_use_mars_sync -eq 1 ]; then - marsadm_do_cmd $secondary_host "down" $res || lib_exit 1 - fi - - synctest_patch_data_device $secondary_host $dev_to_patch $dev_size \ - $patch_length_in_kb $no_of_patches - - if [ $synctest_use_mars_sync -eq 1 ]; then - - synctest_via_mars_sync $primary_host $secondary_host $res $dev \ - "synctime" - else - synctest_via_rsync $secondary_host $primary_host \ - $synctest_data_file "synctime" - fi - echo " synctime = $synctime" - synctimes[$i]=$synctime - - done - echo "use_mars_sync=$synctest_use_mars_sync synctimes = ${synctimes[@]}" - - if [ $synctest_use_mars_sync -eq 0 ]; then - synctest_remove_data_file $synctest_data_file $primary_host \ - $secondary_host - fi -} - -function synctest_remove_data_file -{ - local data_file=$1 - shift - local hosts="$@" host - for host in $hosts; do - lib_vmsg " removing $host:$data_file" - lib_remote_idfile $host "rm -f $data_file" || lib_exit 1 - done -} - -function synctest_determine_write_rate -{ - local host=$1 writer_pid writer_script write_count - local res=${resource_name_list[0]} - lib_rw_start_writing_data_device "writer_pid" "writer_script" 0 0 $res - sleep $synctest_write_time - lib_rw_stop_writing_data_device $writer_script "write_count" - echo "write_count:$write_count, time_write=$synctest_write_time, rate = $(((60 * $write_count ) / $synctest_write_time)) / min" -} - -function synctest_via_rsync -{ - local secondary_host=$1 primary_host=$2 data_file=$3 - local varname_time_waited=$4 - local start=$(date +'%s') time_waited - - synctest_do_rsync $secondary_host $primary_host $data_file - - time_waited=$(($(date +'%s') - $start)) - eval $varname_time_waited=$time_waited -} - -function synctest_do_rsync -{ - local secondary_host=$1 primary_host=$2 data_file=$3 - lib_vmsg " syncing $primary_host:$data_file to $secondary_host" - lib_remote_idfile $secondary_host \ - "rsync -av -e ssh root@$primary_host:$data_file $data_file" || \ - lib_exit 1 -} - -function synctest_via_mars_sync -{ - [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" - local primary_host=$1 secondary_host=$2 res=$3 dev=$4 - local varname_time_waited=$5 - local start_sync writer_pid writer_script writer_start writer_time - - marsadm_do_cmd $secondary_host "invalidate" $res || lib_exit 1 - - - if [ $synctest_parallel_writer -eq 1 ]; then - writer_start=$(date +'%s') - lib_rw_start_writing_data_device "writer_pid" "writer_script" 0 0 $res - fi - - marsadm_do_cmd $secondary_host "up" $res || lib_exit 1 - - lib_wait_until_action_stops "syncstatus" $secondary_host $res \ - $synctest_maxtime_sync \ - $synctest_time_constant_sync \ - $varname_time_waited - - - if [ $synctest_parallel_writer -eq 1 ]; then - local write_count logfile length_logfile time_waited time_sync - - lib_rw_stop_writing_data_device $writer_script "write_count" - writer_time=$(( $(date +'%s') - $writer_start )) - - eval time_sync='$'$varname_time_waited - - echo "write_count:$write_count, time_sync=$time_sync, rate = $(((60 * $write_count ) / $writer_time )) / min" - lib_vmsg " recreating all resources" - resource_recreate_all - - else - lib_rw_compare_checksums $primary_host $secondary_host $dev 0 "" "" - fi -} - -function synctest_generate_data_file -{ - [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" - local primary_host=$1 secondary_host=$2 dev=$3 dev_size=$4 - local data_file=$5 file_size_in_kb=$(($dev_size * 1024 * 1024)) - local host - - lib_vmsg " generating file $primary_host:$data_file ($file_size_in_kb KB) from $dev" - lib_remote_idfile $primary_host \ - "dd if=$dev of=$data_file bs=1024 count=$file_size_in_kb" || \ - lib_exit 1 - synctest_do_rsync $secondary_host $primary_host $data_file - - for host in $primary_host $secondary_host; do - lib_remote_idfile $host "ls -l --full-time $data_file" || lib_exit 1 - done -} - -function synctest_set_sync_modus -{ - local hosts="$@" host - for host in $hosts; do - lib_vmsg " setting fast sync mode to $synctest_fast_sync on $host" - lib_remote_idfile $host \ - "echo $synctest_fast_sync > $synctest_sync_mode_proc_file" \ - || lib_exit 1 - done -} - -function synctest_patch_data_device -{ - [ $# -eq 5 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" - local host=$1 dev=$2 dev_size_in_kb=$(($3 * 1024 * 1024)) - local patch_length_in_kb=$4 no_of_patches=$5 - local offset=0 bs=1024 remaining=$dev_size_in_kb - - while [ $offset -lt $((dev_size_in_kb - $patch_length_in_kb)) ]; do - lib_vmsg " patching $dev at $offset KB with $patch_length_in_kb KB" - lib_remote_idfile $host \ - "yes :$offset: | dd of=$dev bs=$bs skip=$offset count=$patch_length_in_kb" || lib_exit 1 - offset=$(($offset + ($dev_size_in_kb / $no_of_patches))) - done -} diff --git a/test_suite/scripts/modules/76_datadev_full.sh b/test_suite/scripts/modules/76_datadev_full.sh index 5d50013a..122aed33 100644 --- a/test_suite/scripts/modules/76_datadev_full.sh +++ b/test_suite/scripts/modules/76_datadev_full.sh @@ -25,7 +25,7 @@ function datadev_full_run local secondary_host=${main_host_list[1]} local res=${resource_name_list[0]} local dev=$(lv_config_get_lv_device $res) - local data_dev=$(resource_get_name_data_device $res) + local data_dev=$(resource_get_data_device $res) local data_dev_size_orig=$(lv_config_get_lv_size $res) local data_dev_size_new=$(($data_dev_size_orig + $resize_size_to_add)) local mars_data_dev_size_new=$((data_dev_size_new \ diff --git a/test_suite/scripts/modules/78_memleak.sh b/test_suite/scripts/modules/78_memleak.sh new file mode 100644 index 00000000..97b6b15d --- /dev/null +++ b/test_suite/scripts/modules/78_memleak.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +## This test provoked a memleak error in mars versions up to light0.1beta0.12 + +function memleak_run +{ + local primary_host=${main_host_list[0]} + local res=${resource_name_list[0]} + local data_dev=$(resource_get_data_device $res) + local cmd='dd if=/dev/zero of='"$data_dev"' bs=4096 count=1000000 & sleep 1; kill -9 $(jobs -p); maxcount=20; count=0; while test $count -lt $maxcount && ! marsadm secondary '"$res"' ; do date; echo $count; sleep 1; let count+=1; done' + resource_create $primary_host $res + lib_vmsg " starting on $primary_host: $cmd" + lib_remote_idfile $primary_host $cmd + # wait a little for an error file to appear + sleep 3 +} + diff --git a/test_suite/scripts/modules/80_lib_rw.sh b/test_suite/scripts/modules/80_lib_rw.sh index 152d513b..0aaebff2 100644 --- a/test_suite/scripts/modules/80_lib_rw.sh +++ b/test_suite/scripts/modules/80_lib_rw.sh @@ -110,7 +110,8 @@ function lib_rw_start_writing_data_device local res=$5 lib_rw_write_and_delete_loop ${main_host_list[0]} \ ${resource_mount_point_list[$res]}/$lib_rw_file_to_write \ - $(lv_config_get_lv_size ${resource_name_list[0]}) 4 \ + $(lv_config_get_lv_size ${resource_name_list[0]}) \ + $lib_rw_part_of_device_size_written_per_loop \ $varname_pid $varname_script $no_of_loops $sleeptime } @@ -183,9 +184,15 @@ function lib_rw_mount_data_device { local host=$1 dev=$2 mount_point=$3 local res=${resource_name_list[0]} - if ! mount_is_device_mounted $host $dev; then + local mount_point + if ! mount_is_device_mounted $host $dev "mount_point"; then mount_mount $host $dev $mount_point ${resource_fs_type_list[$res]} || \ lib_exit 1 fi } +function lib_wait_until_replay_has_exceeded +{ + local secondary_host=$1 logfile_primary=$2 logfile_length_primary=$3 maxwait=$4 + +} diff --git a/test_suite/scripts/modules/81_lib_linktree.sh b/test_suite/scripts/modules/81_lib_linktree.sh index 7e21210f..cc74aaa6 100644 --- a/test_suite/scripts/modules/81_lib_linktree.sh +++ b/test_suite/scripts/modules/81_lib_linktree.sh @@ -100,7 +100,7 @@ function lib_linktree_check_link else sleep 1 let waited+=1 - lib_vmsg " waited $waited for $link to take $link_value_expected" + lib_vmsg " waited $waited for $link to become $link_value_expected" continue fi else diff --git a/test_suite/scripts/modules/87_apply_fetch.sh b/test_suite/scripts/modules/87_apply_fetch.sh index c36f5bb3..8a35a2a7 100644 --- a/test_suite/scripts/modules/87_apply_fetch.sh +++ b/test_suite/scripts/modules/87_apply_fetch.sh @@ -28,7 +28,7 @@ function apply_fetch_run local secondary_host=${main_host_list[1]} local res=${resource_name_list[0]} local writer_pid writer_script write_count - local logfile length_logfile time_waited + local logfile length_logfile time_waited net_throughput lib_wait_for_initial_end_of_sync $secondary_host $res \ $resource_maxtime_initial_sync \ @@ -44,7 +44,8 @@ function apply_fetch_run lib_wait_until_action_stops "replay" $secondary_host $res \ $apply_fetch_maxtime_apply \ - $apply_fetch_time_constant_apply "time_waited" + $apply_fetch_time_constant_apply \ + "time_waited" 0 "net_throughput" lib_vmsg " ${FUNCNAME[0]}: apply time: $time_waited" marsview_check $secondary_host $res "disk" "Outdated\[.*A.*\]" \ @@ -53,7 +54,8 @@ function apply_fetch_run marsadm_pause_cmd "fetch" $secondary_host $res lib_wait_until_fetch_stops "apply_fetch" $secondary_host $primary_host \ - $res "logfile" "length_logfile" "time_waited" + $res "logfile" "length_logfile" "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited" @@ -66,10 +68,10 @@ function apply_fetch_run lib_wait_until_action_stops "replay" $secondary_host $res \ $apply_fetch_maxtime_apply_after_disconnect \ $apply_fetch_time_constant_apply_after_disconnect \ - "time_waited" + "time_waited" 0 "net_throughput" lib_vmsg " ${FUNCNAME[0]}: apply time: $time_waited" - marsadm_check_warn_file_and_disk_state $secondary_host $res \ + marsadm_check_warnings_and_disk_state $secondary_host $res \ "apply_stopped_after_disconnect" marsview_check $secondary_host $res "repl" "-S-A-" || lib_exit 1 marsadm_do_cmd $secondary_host "connect" $res || lib_exit 1 @@ -79,7 +81,8 @@ function apply_fetch_run lib_wait_until_fetch_stops "apply_fetch" $secondary_host \ $primary_host $res "logfile" \ - "length_logfile" "time_waited" + "length_logfile" "time_waited" 0 \ + "net_throughput" lib_vmsg " ${FUNCNAME[0]}: fetch time: $time_waited" file_handling_check_equality_of_file_lengths $logfile \ diff --git a/test_suite/scripts/modules/95_marsadm.sh b/test_suite/scripts/modules/95_marsadm.sh index 7e1921e8..ed8bdeca 100644 --- a/test_suite/scripts/modules/95_marsadm.sh +++ b/test_suite/scripts/modules/95_marsadm.sh @@ -92,8 +92,8 @@ function marsadm_check_post_condition_role_switch if [ "$role_act" != "$role_req" ]; then lib_exit 1 "role expected = $role_req != $role_act = role found" fi - lib_vmsg " role = $role_act, trying ls $(resource_get_name_data_device $res) on $host" - lib_remote_idfile $host "ls -l --full-time $(resource_get_name_data_device $res)" + lib_vmsg " role = $role_act, trying ls $(resource_get_data_device $res) on $host" + lib_remote_idfile $host "ls -l --full-time $(resource_get_data_device $res)" rc=$? case $role_act in # ((( primary) if [ $rc -ne 0 ]; then @@ -180,13 +180,15 @@ function marsadm_set_proc_sys_mars_parameter lib_remote_idfile $host "echo $param_value >$dir/$param" || lib_exit 1 } -function marsadm_check_warn_file_and_disk_state +function marsadm_check_warnings_and_disk_state { local host=$1 res=$2 situation="$3" + # wait a little for the files required + sleep 2 case $situation in # (( apply_stopped_after_disconnect) local link_value not_applied restlen_in_warn_file - local warn_file="${resource_dir_list[$res]}/2.warn.status" + local warn_file="$cluster_debugfile" local link=$(lib_linktree_get_res_host_linkname $host $res "replay") link_value=$(lib_remote_idfile $host "readlink $link") || lib_exit 1 diff --git a/test_suite/scripts/modules/99_mount.sh b/test_suite/scripts/modules/99_mount.sh index 092dfcf5..be33b17c 100644 --- a/test_suite/scripts/modules/99_mount.sh +++ b/test_suite/scripts/modules/99_mount.sh @@ -33,17 +33,30 @@ function mount_mount { [ $# -eq 4 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local host=$1 dev="$2" mount_point=$3 fs_type=$4 - lib_vmsg " mounting dev $dev on $mount_point on $host" + lib_vmsg " mounting dev $dev (type $fs_type) on $mount_point on $host" lib_remote_idfile $host mount -t $fs_type $dev $mount_point return $? } function mount_is_device_mounted { - local host=$1 dev=$2 rc - lib_vmsg " checking whether $dev is mounted on $host" - lib_remote_idfile $host "mount | grep '^$dev on'" + [ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" + local host=$1 dev=$2 varname_mountpoint=$3 rc + local mount_out devname_in_mounttab + # in the mount table the device appears under the device mapper name + # /dev/mapper/... + lib_vmsg " checking whether $host:$dev is 'device mapped'" + devname_in_mounttab=$(lv_config_get_dm_dev $host $dev) rc=$? + if [ $rc -ne 0 ]; then + devname_in_mounttab=$dev + fi + lib_vmsg " checking whether $devname_in_mounttab is mounted on $host" + mount_out=($(lib_remote_idfile $host "mount | grep '^$devname_in_mounttab on'")) + rc=$? + if [ $rc -eq 0 ]; then + eval $varname_mountpoint=${mount_out[2]} + fi return $rc } @@ -61,20 +74,37 @@ function mount_mount_data_device local res_no=${1:-0} local res=${resource_name_list[$res_no]} local host=${main_host_list[0]} - local dev=$(resource_get_name_data_device $res) + local dev=$(resource_get_data_device $res) local mount_point=${resource_mount_point_list[$res]} + + lib_remote_check_device_fs $host $dev ${resource_fs_type_list[$res]} lib_rw_mount_data_device $host $dev $mount_point } -function mount_umount_data_device +function mount_umount_data_device_all { local res_no=${1:-0} + local host local res=${resource_name_list[$res_no]} - local host=${main_host_list[0]} - local dev=$(resource_get_name_data_device $res) + local dev=$(resource_get_data_device $res) local mount_point=${resource_mount_point_list[$res]} - if mount_is_dir_mountpoint $host $mount_point; then - mount_umount $host $dev $mount_point - fi + for host in ${main_host_list[@]}; do + if mount_is_dir_mountpoint $host $mount_point; then + local maxwait=5 waited=0 rc + while true;do + mount_umount $host $dev $mount_point + rc=$? + if [ $rc -eq 0 ];then + break + fi + sleep 1 + let waited+=1 + lib_vmsg " waited $waited for unmounting $host:$mount_point" + if [ $waited -eq $maxwait ]; then + lib_exit 1 "maxwait $maxwait exceeded" + fi + done + fi + done } diff --git a/test_suite/scripts/modules/lib.sh b/test_suite/scripts/modules/lib.sh index 2339866a..f55deba3 100644 --- a/test_suite/scripts/modules/lib.sh +++ b/test_suite/scripts/modules/lib.sh @@ -116,6 +116,17 @@ check_installed "$check_always_list" ##################################################################### +# miscellanous + +## rounds to GiB +function lib_round_to_gb +{ + local number=$1 + echo $((($number + (512 * 1024 * 1024)) / (1024 * 1024 * 1024))) +} + +##################################################################### + # helper for sourcing other config files (may reside in parents of cwd) function source_config @@ -247,28 +258,27 @@ function lib_remote_check_device_fs { [ $# -eq 3 ] || lib_exit 1 "wrong number $# of arguments (args = $*)" local host=$1 dev=$2 fs_type=$3 - lib_vmsg " checking device $dev on host $host" - lib_remote_idfile $host "lvdisplay -C --noheadings -o lv_name $dev" \ - || lib_exit 1 - lib_vmsg " checking whether $host:$dev contains a filesystem" - blkid_out=$(lib_remote_idfile $host "blkid -p -u filesystem $dev") + local tmp_dir=/mnt/mars_tmp_mountpoint + lib_vmsg " checking existence of directory $host:$tmp_dir" + lib_remote_idfile $host "if test ! -d $tmp_dir; then mkdir $tmp_dir;fi" \ + || lib_exit 1 + lib_vmsg " checking whether $host:$dev is mountable as $fs_type filesystem on $tmp_dir" + lib_remote_idfile $host mount -t $fs_type $dev $tmp_dir rc=$? if [ $rc -eq 0 ]; then - if ! expr "$blkid_out" : '.*\(TYPE="'"$fs_type"'"\)'; then - rc=1 - fi + mount_umount $host $dev $tmp_dir || lib_exit 1 + return fi - if [ $rc -ne 0 ]; then - if mount_is_dir_mountpoint $host $main_mars_directory; then - mount_umount $host "device_does_not_matter" $main_mars_directory - fi - lib_vmsg " creating $fs_type filesystem on $dev" - lib_remote_idfile $host "mkfs.$fs_type ${lv_config_mkfs_option_list[$fs_type]} $dev" || lib_exit 1 - if [ -n "${lv_config_fs_type_tune_cmd_list[$fs_type]}" ];then - local cmd=${lv_config_fs_type_tune_cmd_list[$fs_type]//$dev} - lib_vmsg " tuning $dev on $host: $cmd" - lib_remote_idfile $host "$cmd" || lib_exit 1 - fi + local mount_point + if mount_is_device_mounted $host $dev "mount_point"; then + mount_umount $host $dev $mount_point + fi + lib_vmsg " creating $fs_type filesystem on $dev" + lib_remote_idfile $host "mkfs.$fs_type ${lv_config_mkfs_option_list[$fs_type]} $dev" || lib_exit 1 + if [ -n "${lv_config_fs_type_tune_cmd_list[$fs_type]}" ];then + local cmd=${lv_config_fs_type_tune_cmd_list[$fs_type]//$dev} + lib_vmsg " tuning $dev on $host: $cmd" + lib_remote_idfile $host "$cmd" || lib_exit 1 fi } diff --git a/test_suite/scripts/start_test.sh b/test_suite/scripts/start_test.sh index 99b8cc66..40fdcf73 100755 --- a/test_suite/scripts/start_test.sh +++ b/test_suite/scripts/start_test.sh @@ -158,7 +158,7 @@ while (( resume )); do eval "$to_start" || { echo "Replay failure $?"; exit -1; } fi echo "==> $(date) Finished." - ) || { echo "Failure $?"; exit -1; } + ) || { rc=$?; echo "Failure $rc"; exit -1; } echo "===============================================================" echo "" # (( resume++ )) diff --git a/test_suite/synctest.out.20130718150923 b/test_suite/synctest.out.20130718150923 new file mode 100644 index 00000000..b3be1a3f --- /dev/null +++ b/test_suite/synctest.out.20130718150923 @@ -0,0 +1,4484 @@ +Sourcing lib.sh +Sourcing config file /home/fl/mars/test_suite/default-main.conf +Sourcing module main +Sourcing config file /home/fl/mars/test_suite/default-checkout_mars.conf +Sourcing module checkout_mars +Sourcing config file /home/fl/mars/test_suite/default-make_mars.conf +Sourcing module make_mars +Sourcing config file /home/fl/mars/test_suite/default-grub.conf +Sourcing module grub +Sourcing config file /home/fl/mars/test_suite/default-install_mars.conf +Sourcing module install_mars +Sourcing config file /home/fl/mars/test_suite/default-lv_config.conf +Sourcing module lv_config +Sourcing config file /home/fl/mars/test_suite/default-cluster.conf +Sourcing module cluster +Sourcing config file /home/fl/mars/test_suite/default-resource.conf +Sourcing module resource +Sourcing config file /home/fl/mars/test_suite/default-file_handling.conf +Sourcing module file_handling +Sourcing config file /home/fl/mars/test_suite/default-remote_dev.conf +Sourcing module remote_dev +Sourcing config file /home/fl/mars/test_suite/default-resize.conf +Sourcing module resize +Sourcing config file /home/fl/mars/test_suite/default-logrotate.conf +Sourcing module logrotate +Sourcing config file /home/fl/mars/test_suite/default-crash.conf +Sourcing module crash +Sourcing config file /home/fl/mars/test_suite/default-file_destroy.conf +Sourcing module file_destroy +Sourcing config file /home/fl/mars/test_suite/default-lib_wait.conf +Sourcing module lib_wait +Sourcing config file /home/fl/mars/test_suite/default-synctest.conf +Sourcing module synctest +Sourcing config file /home/fl/mars/test_suite/default-lib_rw.conf +Sourcing module lib_rw +Sourcing config file /home/fl/mars/test_suite/default-lib_linktree.conf +Sourcing module lib_linktree +Sourcing config file /home/fl/mars/test_suite/default-apply_fetch.conf +Sourcing module apply_fetch +Sourcing config file /home/fl/mars/test_suite/default-marsadm.conf +Sourcing module marsadm +Sourcing config file /home/fl/mars/test_suite/default-marsview.conf +Sourcing module marsview +Sourcing config file /home/fl/mars/test_suite/default-mount.conf +Sourcing module mount +Scanning directory structure starting from /home/fl/mars/test_suite/test_cases/perf/sync +Ignoring inner directory . +Ignoring inner directory ./fast_sync + +=============================================================== +======== ./fast_sync/no_parallel_writer +Sourcing config file /home/fl/mars/test_suite/user_modules.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/fast_sync.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/no_parallel_writer.conf +==> Thu Jul 18 12:56:59 CEST 2013 Starting .fast_sync.no_parallel_writer +calling synctest_run + setting fast sync mode to 1 on istore-test-bs7 + setting fast sync mode to 1 on istore-test-bap7 + patchlength 1, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.000464674 s, 2.2 MB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.723313 s, 1.4 kB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 1.49603 s, 0.7 kB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.06368 s, 0.5 kB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.76542 s, 0.4 kB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.10202 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.74255 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 4.37669 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 5.03635 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 5.60976 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 2097150 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 6.19924 s, 0.2 kB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 1139511296 + waited 2 secs for syncstatus to stop. old = 1139511296, act = 1730367488 + waited 3 secs for syncstatus to stop. old = 1730367488, act = 2147483648 + waited 4 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 5 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 6 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 7 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 8 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 9 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 10 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 11 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 12 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 13 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 14 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 15 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 16 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 17 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 18 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 19 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 20 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 21 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 22 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + synctime = 2 + patchlength 50000, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 0.783988 s, 65.3 MB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 1.46724 s, 34.9 MB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.09204 s, 24.5 MB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.79584 s, 18.3 MB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.46097 s, 14.8 MB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 4.10661 s, 12.5 MB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 4.43251 s, 11.6 MB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.07834 s, 10.1 MB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.72625 s, 8.9 MB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 6.32215 s, 8.1 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 766910464 + waited 2 secs for syncstatus to stop. old = 766910464, act = 1333927936 + waited 3 secs for syncstatus to stop. old = 1333927936, act = 1813045248 + waited 4 secs for syncstatus to stop. old = 1813045248, act = 2147483648 + waited 5 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 6 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 7 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 8 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 9 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 10 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 11 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 12 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 13 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 14 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 15 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 16 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 17 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 18 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 19 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 20 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 21 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 22 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 23 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + synctime = 3 + patchlength 500000, no of patches 4 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 7.26034 s, 70.5 MB/s + patching /dev/vg-mars/lv-2 at 524288 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 8.66746 s, 59.1 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 10.1394 s, 50.5 MB/s + patching /dev/vg-mars/lv-2 at 1572864 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 11.6697 s, 43.9 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 110112768 + waited 2 secs for syncstatus to stop. old = 110112768, act = 164581376 + waited 3 secs for syncstatus to stop. old = 164581376, act = 196251648 + waited 4 secs for syncstatus to stop. old = 196251648, act = 258015232 + waited 5 secs for syncstatus to stop. old = 258015232, act = 302972928 + waited 6 secs for syncstatus to stop. old = 302972928, act = 358977536 + waited 7 secs for syncstatus to stop. old = 358977536, act = 402984960 + waited 8 secs for syncstatus to stop. old = 402984960, act = 438890496 + waited 9 secs for syncstatus to stop. old = 438890496, act = 500793344 + waited 10 secs for syncstatus to stop. old = 500793344, act = 914083840 + waited 11 secs for syncstatus to stop. old = 914083840, act = 1239302144 + waited 12 secs for syncstatus to stop. old = 1239302144, act = 1702850560 + waited 13 secs for syncstatus to stop. old = 1702850560, act = 2147483648 + waited 14 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 15 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 16 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 17 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 18 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 19 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 20 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 21 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 22 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 23 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 24 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 25 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 26 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 27 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 28 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 29 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 30 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 31 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 32 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + synctime = 12 + patchlength 1000000, no of patches 2 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 14.2555 s, 71.8 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 17.1111 s, 59.8 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 123674624 + waited 2 secs for syncstatus to stop. old = 123674624, act = 178495488 + waited 3 secs for syncstatus to stop. old = 178495488, act = 215035904 + waited 4 secs for syncstatus to stop. old = 215035904, act = 273625088 + waited 5 secs for syncstatus to stop. old = 273625088, act = 327958528 + waited 6 secs for syncstatus to stop. old = 327958528, act = 371314688 + waited 7 secs for syncstatus to stop. old = 371314688, act = 432263168 + waited 8 secs for syncstatus to stop. old = 432263168, act = 479723520 + waited 9 secs for syncstatus to stop. old = 479723520, act = 519962624 + waited 10 secs for syncstatus to stop. old = 519962624, act = 579706880 + waited 11 secs for syncstatus to stop. old = 579706880, act = 631492608 + waited 12 secs for syncstatus to stop. old = 631492608, act = 687972352 + waited 13 secs for syncstatus to stop. old = 687972352, act = 744652800 + waited 14 secs for syncstatus to stop. old = 744652800, act = 783085568 + waited 15 secs for syncstatus to stop. old = 783085568, act = 823758848 + waited 16 secs for syncstatus to stop. old = 823758848, act = 885567488 + waited 17 secs for syncstatus to stop. old = 885567488, act = 940814336 + waited 18 secs for syncstatus to stop. old = 940814336, act = 982097920 + waited 19 secs for syncstatus to stop. old = 982097920, act = 1165172736 + waited 20 secs for syncstatus to stop. old = 1165172736, act = 1630482432 + waited 21 secs for syncstatus to stop. old = 1630482432, act = 2147483648 + waited 22 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 23 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 24 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 25 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 26 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 27 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 28 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 29 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 30 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 31 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 32 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 33 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 34 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 35 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 36 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 37 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 38 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 39 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 40 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 3794737506 2147483648 /dev/vg-mars/lv-2 + synctime = 20 +use_mars_sync=1 synctimes = 2 3 12 20 +==> Thu Jul 18 13:10:17 CEST 2013 Finished. +=============================================================== + + +=============================================================== +======== ./fast_sync/parallel_writer +Sourcing config file /home/fl/mars/test_suite/user_modules.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/fast_sync.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/parallel_writer.conf +==> Thu Jul 18 13:10:17 CEST 2013 Starting .fast_sync.parallel_writer +calling synctest_run + setting fast sync mode to 1 on istore-test-bs7 + setting fast sync mode to 1 on istore-test-bap7 + checking whether /dev/mars/lv-2 is mounted on istore-test-bs7 +/dev/mars/lv-2 on /mnt/test type ext3 (rw) + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 21616 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 21616 1 0 13:10 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 21616 has errors in /tmp/write_and_delete.sh.13181.err + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=21616) on istore-test-bs7 + write_count: 12 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:12, time_write=20, rate = 36 / min + patchlength 1, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.00289405 s, 354 kB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.72321 s, 1.4 kB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 1.49907 s, 0.7 kB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.26554 s, 0.5 kB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.86417 s, 0.4 kB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.29488 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.89167 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 4.71609 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 5.28264 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 6.03181 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 2097150 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 6.99138 s, 0.1 kB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 23022 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 23022 1 0 13:11 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 23022 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 173834240 + waited 2 secs for syncstatus to stop. old = 173834240, act = 196009984 + waited 3 secs for syncstatus to stop. old = 196009984, act = 266928128 + waited 4 secs for syncstatus to stop. old = 266928128, act = 272138240 + waited 5 secs for syncstatus to stop. old = 272138240, act = 302510080 + waited 6 secs for syncstatus to stop. old = 302510080, act = 309293056 + waited 7 secs for syncstatus to stop. old = 309293056, act = 311730176 + waited 8 secs for syncstatus to stop. old = 311730176, act = 340934656 + waited 9 secs for syncstatus to stop. old = 340934656, act = 353140736 + waited 10 secs for syncstatus to stop. old = 353140736, act = 372502528 + waited 11 secs for syncstatus to stop. old = 372502528, act = 375439360 + waited 12 secs for syncstatus to stop. old = 375439360, act = 412676096 + waited 13 secs for syncstatus to stop. old = 412676096, act = 452603904 + waited 14 secs for syncstatus to stop. old = 452603904, act = 539324416 + waited 15 secs for syncstatus to stop. old = 539324416, act = 584572928 + waited 16 secs for syncstatus to stop. old = 584572928, act = 639426560 + waited 17 secs for syncstatus to stop. old = 639426560, act = 654315520 + waited 18 secs for syncstatus to stop. old = 654315520, act = 654315520 + waited 19 secs for syncstatus to stop. old = 654315520, act = 681156608 + waited 20 secs for syncstatus to stop. old = 681156608, act = 723959808 + waited 21 secs for syncstatus to stop. old = 723959808, act = 766918656 + waited 22 secs for syncstatus to stop. old = 766918656, act = 813277184 + waited 23 secs for syncstatus to stop. old = 813277184, act = 844632064 + waited 24 secs for syncstatus to stop. old = 844632064, act = 884224000 + waited 25 secs for syncstatus to stop. old = 884224000, act = 884224000 + waited 26 secs for syncstatus to stop. old = 884224000, act = 934014976 + waited 27 secs for syncstatus to stop. old = 934014976, act = 934014976 + waited 28 secs for syncstatus to stop. old = 934014976, act = 934014976 + waited 29 secs for syncstatus to stop. old = 934014976, act = 973840384 + waited 30 secs for syncstatus to stop. old = 973840384, act = 1048584192 + waited 31 secs for syncstatus to stop. old = 1048584192, act = 1048584192 + waited 32 secs for syncstatus to stop. old = 1048584192, act = 1082138624 + waited 33 secs for syncstatus to stop. old = 1082138624, act = 1114075136 + waited 34 secs for syncstatus to stop. old = 1114075136, act = 1117876224 + waited 35 secs for syncstatus to stop. old = 1117876224, act = 1141252096 + waited 36 secs for syncstatus to stop. old = 1141252096, act = 1149247488 + waited 37 secs for syncstatus to stop. old = 1149247488, act = 1158963200 + waited 38 secs for syncstatus to stop. old = 1158963200, act = 1182801920 + waited 39 secs for syncstatus to stop. old = 1182801920, act = 1216356352 + waited 40 secs for syncstatus to stop. old = 1216356352, act = 1216364544 + waited 41 secs for syncstatus to stop. old = 1216364544, act = 1236144128 + waited 42 secs for syncstatus to stop. old = 1236144128, act = 1248178176 + waited 43 secs for syncstatus to stop. old = 1248178176, act = 1297801216 + waited 44 secs for syncstatus to stop. old = 1297801216, act = 1323376640 + waited 45 secs for syncstatus to stop. old = 1323376640, act = 1362214912 + waited 46 secs for syncstatus to stop. old = 1362214912, act = 1431773184 + waited 47 secs for syncstatus to stop. old = 1431773184, act = 1511280640 + waited 48 secs for syncstatus to stop. old = 1511280640, act = 1585455104 + waited 49 secs for syncstatus to stop. old = 1585455104, act = 1625235456 + waited 50 secs for syncstatus to stop. old = 1625235456, act = 1697656832 + waited 51 secs for syncstatus to stop. old = 1697656832, act = 1746935808 + waited 52 secs for syncstatus to stop. old = 1746935808, act = 1769689088 + waited 53 secs for syncstatus to stop. old = 1769689088, act = 1788985344 + waited 54 secs for syncstatus to stop. old = 1788985344, act = 1816199168 + waited 55 secs for syncstatus to stop. old = 1816199168, act = 1832153088 + waited 56 secs for syncstatus to stop. old = 1832153088, act = 1861468160 + waited 57 secs for syncstatus to stop. old = 1861468160, act = 1861468160 + waited 58 secs for syncstatus to stop. old = 1861468160, act = 1861468160 + waited 59 secs for syncstatus to stop. old = 1861468160, act = 1861468160 + waited 60 secs for syncstatus to stop. old = 1861468160, act = 1861468160 + waited 61 secs for syncstatus to stop. old = 1861468160, act = 1861468160 + waited 62 secs for syncstatus to stop. old = 1861468160, act = 1873199104 + waited 63 secs for syncstatus to stop. old = 1873199104, act = 1940873216 + waited 64 secs for syncstatus to stop. old = 1940873216, act = 1951264768 + waited 65 secs for syncstatus to stop. old = 1951264768, act = 1975910400 + waited 66 secs for syncstatus to stop. old = 1975910400, act = 1981349888 + waited 67 secs for syncstatus to stop. old = 1981349888, act = 2007826432 + waited 68 secs for syncstatus to stop. old = 2007826432, act = 2015371264 + waited 69 secs for syncstatus to stop. old = 2015371264, act = 2031943680 + waited 70 secs for syncstatus to stop. old = 2031943680, act = 2040070144 + waited 71 secs for syncstatus to stop. old = 2040070144, act = 2058186752 + waited 72 secs for syncstatus to stop. old = 2058186752, act = 2058186752 + waited 73 secs for syncstatus to stop. old = 2058186752, act = 2083848192 + waited 74 secs for syncstatus to stop. old = 2083848192, act = 2094071808 + waited 75 secs for syncstatus to stop. old = 2094071808, act = 2114658304 + waited 76 secs for syncstatus to stop. old = 2114658304, act = 2117402624 + waited 77 secs for syncstatus to stop. old = 2117402624, act = 2147483648 + waited 78 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 79 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 80 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 81 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 82 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 83 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 84 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 85 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 86 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 87 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 88 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 89 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 90 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 91 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 92 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 93 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 94 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 95 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 96 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=23022) on istore-test-bs7 + write_count: 47 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:47, time_sync=306, rate = 9 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22596395008, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22632521728, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22596395008 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22669787136, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22632521728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22702149632, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22669787136 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22738034688, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22702149632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22778146816, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22738034688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22817325056, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22778146816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22855659520, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22817325056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22894338048, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22855659520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22932541440, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22894338048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:22968500224, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22932541440 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23003664384, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:22968500224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23040962560, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23003664384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23081484288, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23040962560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23119912960, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23081484288 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23153975296, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23119912960 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23188709376, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23153975296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23222497280, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23188709376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23257731072, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23222497280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23293800448, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23257731072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23332982784, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23293800448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23370928128, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23332982784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23407824896, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23370928128 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23441252352, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23407824896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23476363264, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23441252352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23509868544, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23476363264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23548276736, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23509868544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23586263040, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23548276736 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23624347648, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23586263040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23662977024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23624347648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23700992000, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23662977024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23740674048, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23700992000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23778918400, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23740674048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23816658944, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23778918400 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23853928448, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23816658944 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23891640320, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23853928448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23930896384, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23891640320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:23967895552, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23930896384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24004161536, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:23967895552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24040583168, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24004161536 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24079671296, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24040583168 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24114298880, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24079671296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24150196224, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24114298880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24184561664, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24150196224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24223600640, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24184561664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24259600384, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24223600640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24298786816, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24259600384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24336252928, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24298786816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24373186560, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24336252928 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24411172864, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24373186560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24448626688, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24411172864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24488214528, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24448626688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24524673024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24488214528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24564449280, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24524673024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24601690112, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24564449280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24641368064, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24601690112 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24680378368, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24641368064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24720625664, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24680378368 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24755347456, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24720625664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24790114304, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24755347456 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24828305408, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24790114304 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24864727040, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24828305408 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24902762496, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24864727040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24936534016, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24902762496 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:24974761984, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24936534016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25013493760, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:24974761984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25051791360, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25013493760 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25090351104, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25051791360 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25126842368, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25090351104 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25160732672, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25126842368 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25199882240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25160732672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25237798912, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25199882240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25276411904, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25237798912 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25312231424, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25276411904 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25347559424, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25312231424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25381670912, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25347559424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25419292672, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25381670912 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25454342144, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25419292672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25488875520, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25454342144 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25525923840, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25488875520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25565569024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25525923840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25603657728, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25565569024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25644134400, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25603657728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25680461180, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25644134400 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25703821312, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25680461180 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25741996032, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25703821312 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25780281344, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25741996032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 88 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25816666112, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25780281344 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 89 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25856937984, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25816666112 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 90 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25897062400, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25856937984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 91 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25936625664, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25897062400 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 92 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:25977315328, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25936625664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 93 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26011967488, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:25977315328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 94 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26051121152, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26011967488 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 95 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26090307584, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26051121152 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 96 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26129383424, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26090307584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 97 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26167164928, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26129383424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 98 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26201686016, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26167164928 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 99 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26241445888, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26201686016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 100 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26279653376, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26241445888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 101 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26314829824, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26279653376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 102 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26353733632, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26314829824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 103 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26388934656, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26353733632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 104 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26426048512, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26388934656 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 105 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26466373632, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26426048512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 106 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26501795840, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26466373632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 107 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26537934848, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26501795840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 108 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26575699968, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26537934848 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 109 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26610917376, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26575699968 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 110 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26650808320, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26610917376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 111 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26687950848, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26650808320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 112 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26726596608, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26687950848 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 113 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26764349440, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26726596608 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 114 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26802221056, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26764349440 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 115 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26802221056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 116 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 117 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 118 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 119 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 120 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 121 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 122 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 123 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 124 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:26834166260 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000001-istore-test-bs7,26834166260,0 + waited 2 secs for replay to stop. old = log-000000001-istore-test-bs7,26834166260,0, act = log-000000001-istore-test-bs7,26834166260,0 + waited 3 secs for replay to stop. old = log-000000001-istore-test-bs7,26834166260,0, act = log-000000001-istore-test-bs7,26834166260,0 + waited 4 secs for replay to stop. old = log-000000001-istore-test-bs7,26834166260,0, act = log-000000001-istore-test-bs7,26834166260,0 + waited 5 secs for replay to stop. old = log-000000001-istore-test-bs7,26834166260,0, act = log-000000001-istore-test-bs7,26834166260,0 + synctime = 76 + patchlength 50000, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 0.758433 s, 67.5 MB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 1.45809 s, 35.1 MB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.15013 s, 23.8 MB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.79998 s, 18.3 MB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.54089 s, 14.5 MB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.8819 s, 13.2 MB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 4.67739 s, 10.9 MB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.10873 s, 10.0 MB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.71463 s, 9.0 MB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 6.35191 s, 8.1 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 26546 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 26546 1 0 13:23 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 26546 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 263483392 + waited 2 secs for syncstatus to stop. old = 263483392, act = 359043072 + waited 3 secs for syncstatus to stop. old = 359043072, act = 567607296 + waited 4 secs for syncstatus to stop. old = 567607296, act = 689618944 + waited 5 secs for syncstatus to stop. old = 689618944, act = 812597248 + waited 6 secs for syncstatus to stop. old = 812597248, act = 857014272 + waited 7 secs for syncstatus to stop. old = 857014272, act = 953540608 + waited 8 secs for syncstatus to stop. old = 953540608, act = 956657664 + waited 9 secs for syncstatus to stop. old = 956657664, act = 1006960640 + waited 10 secs for syncstatus to stop. old = 1006960640, act = 1039384576 + waited 11 secs for syncstatus to stop. old = 1039384576, act = 1064763392 + waited 12 secs for syncstatus to stop. old = 1064763392, act = 1064763392 + waited 13 secs for syncstatus to stop. old = 1064763392, act = 1127092224 + waited 14 secs for syncstatus to stop. old = 1127092224, act = 1131851776 + waited 15 secs for syncstatus to stop. old = 1131851776, act = 1142059008 + waited 16 secs for syncstatus to stop. old = 1142059008, act = 1264058368 + waited 17 secs for syncstatus to stop. old = 1264058368, act = 1294729216 + waited 18 secs for syncstatus to stop. old = 1294729216, act = 1294729216 + waited 19 secs for syncstatus to stop. old = 1294729216, act = 1331908608 + waited 20 secs for syncstatus to stop. old = 1331908608, act = 1331908608 + waited 21 secs for syncstatus to stop. old = 1331908608, act = 1387470848 + waited 22 secs for syncstatus to stop. old = 1387470848, act = 1387470848 + waited 23 secs for syncstatus to stop. old = 1387470848, act = 1413312512 + waited 24 secs for syncstatus to stop. old = 1413312512, act = 1413312512 + waited 25 secs for syncstatus to stop. old = 1413312512, act = 1453072384 + waited 26 secs for syncstatus to stop. old = 1453072384, act = 1502896128 + waited 27 secs for syncstatus to stop. old = 1502896128, act = 1534607360 + waited 28 secs for syncstatus to stop. old = 1534607360, act = 1534607360 + waited 29 secs for syncstatus to stop. old = 1534607360, act = 1534607360 + waited 30 secs for syncstatus to stop. old = 1534607360, act = 1534607360 + waited 31 secs for syncstatus to stop. old = 1534607360, act = 1534607360 + waited 32 secs for syncstatus to stop. old = 1534607360, act = 1535836160 + waited 33 secs for syncstatus to stop. old = 1535836160, act = 1552052224 + waited 34 secs for syncstatus to stop. old = 1552052224, act = 1552052224 + waited 35 secs for syncstatus to stop. old = 1552052224, act = 1599016960 + waited 36 secs for syncstatus to stop. old = 1599016960, act = 1663205376 + waited 37 secs for syncstatus to stop. old = 1663205376, act = 1759248384 + waited 38 secs for syncstatus to stop. old = 1759248384, act = 1786101760 + waited 39 secs for syncstatus to stop. old = 1786101760, act = 1813188608 + waited 40 secs for syncstatus to stop. old = 1813188608, act = 1831137280 + waited 41 secs for syncstatus to stop. old = 1831137280, act = 1831137280 + waited 42 secs for syncstatus to stop. old = 1831137280, act = 1879056384 + waited 43 secs for syncstatus to stop. old = 1879056384, act = 1879056384 + waited 44 secs for syncstatus to stop. old = 1879056384, act = 1887604736 + waited 45 secs for syncstatus to stop. old = 1887604736, act = 1889116160 + waited 46 secs for syncstatus to stop. old = 1889116160, act = 1936449536 + waited 47 secs for syncstatus to stop. old = 1936449536, act = 1946165248 + waited 48 secs for syncstatus to stop. old = 1946165248, act = 1946165248 + waited 49 secs for syncstatus to stop. old = 1946165248, act = 1946165248 + waited 50 secs for syncstatus to stop. old = 1946165248, act = 1950502912 + waited 51 secs for syncstatus to stop. old = 1950502912, act = 1952059392 + waited 52 secs for syncstatus to stop. old = 1952059392, act = 1952370688 + waited 53 secs for syncstatus to stop. old = 1952370688, act = 1956184064 + waited 54 secs for syncstatus to stop. old = 1956184064, act = 1956823040 + waited 55 secs for syncstatus to stop. old = 1956823040, act = 1959424000 + waited 56 secs for syncstatus to stop. old = 1959424000, act = 1961172992 + waited 57 secs for syncstatus to stop. old = 1961172992, act = 1964052480 + waited 58 secs for syncstatus to stop. old = 1964052480, act = 1964838912 + waited 59 secs for syncstatus to stop. old = 1964838912, act = 1968504832 + waited 60 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 61 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 62 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 63 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 64 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 65 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 66 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 67 secs for syncstatus to stop. old = 1968504832, act = 1968504832 + waited 68 secs for syncstatus to stop. old = 1968504832, act = 1968541696 + waited 69 secs for syncstatus to stop. old = 1968541696, act = 1972215808 + waited 70 secs for syncstatus to stop. old = 1972215808, act = 1973702656 + waited 71 secs for syncstatus to stop. old = 1973702656, act = 1976221696 + waited 72 secs for syncstatus to stop. old = 1976221696, act = 1980305408 + waited 73 secs for syncstatus to stop. old = 1980305408, act = 1983152128 + waited 74 secs for syncstatus to stop. old = 1983152128, act = 1985544192 + waited 75 secs for syncstatus to stop. old = 1985544192, act = 1985544192 + waited 76 secs for syncstatus to stop. old = 1985544192, act = 1990918144 + waited 77 secs for syncstatus to stop. old = 1990918144, act = 2019016704 + waited 78 secs for syncstatus to stop. old = 2019016704, act = 2063151104 + waited 79 secs for syncstatus to stop. old = 2063151104, act = 2078580736 + waited 80 secs for syncstatus to stop. old = 2078580736, act = 2147483648 + waited 81 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 82 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 83 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 84 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 85 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 86 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 87 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 88 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 89 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 90 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 91 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 92 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 93 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 94 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 95 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 96 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 97 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 98 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 99 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=26546) on istore-test-bs7 + write_count: 46 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:46, time_sync=354, rate = 7 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:28982988800, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29019009024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:28982988800 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29058670592, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29019009024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29095616512, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29058670592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29134946304, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29095616512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29174054912, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29134946304 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29213097984, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29174054912 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29250433024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29213097984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29290569728, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29250433024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29329723392, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29290569728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29365821440, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29329723392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29407084544, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29365821440 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29443383296, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29407084544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29477433344, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29443383296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29514018816, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29477433344 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29551120384, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29514018816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29585371136, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29551120384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29625470976, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29585371136 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29650173952, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29625470976 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29683896320, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29650173952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29720866816, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29683896320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29758156800, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29720866816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29792952320, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29758156800 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29831409664, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29792952320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29867368448, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29831409664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29906751488, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29867368448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29943435264, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29906751488 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:29982146560, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29943435264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30019874816, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:29982146560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30059065344, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30019874816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30097276928, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30059065344 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30135209984, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30097276928 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30174253056, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30135209984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30210584576, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30174253056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30245904384, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30210584576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30285578240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30245904384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30323982336, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30285578240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30361575424, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30323982336 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30395498496, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30361575424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30433185792, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30395498496 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30472675328, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30433185792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30512529408, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30472675328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30551818240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30512529408 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30587977728, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30551818240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30625615872, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30587977728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30661296128, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30625615872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30698229760, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30661296128 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30734544896, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30698229760 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30773706752, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30734544896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30809067520, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30773706752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30846537728, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30809067520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30884474880, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30846537728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30923788288, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30884474880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:30967222272, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30923788288 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31013453824, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:30967222272 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31048941568, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31013453824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31084408832, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31048941568 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31120109568, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31084408832 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31216484352, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31120109568 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31254597632, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31216484352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31291445248, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31254597632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31329681408, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31291445248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31366868992, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31329681408 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31401418752, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31366868992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31493726208, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31401418752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31533924352, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31493726208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31571513344, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31533924352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31610388480, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31571513344 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31648768000, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31610388480 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31693340672, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31648768000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31728840704, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31693340672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31771209728, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31728840704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31809871872, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31771209728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31843332096, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31809871872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31881498624, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31843332096 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31920164864, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31881498624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31958130688, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31920164864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:31995629568, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31958130688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32031514624, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:31995629568 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32072065024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32031514624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32112037888, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32072065024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32151314432, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32112037888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32190652416, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32151314432 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32227753984, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32190652416 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32266088448, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32227753984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32302239744, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32266088448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32339943424, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32302239744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 88 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32377794560, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32339943424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 89 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32415617024, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32377794560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 90 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32450654208, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32415617024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 91 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32485822464, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32450654208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 92 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32520323072, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32485822464 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 93 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32559017984, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32520323072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 94 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32597336064, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32559017984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 95 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32635502592, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32597336064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 96 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32674201600, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32635502592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 97 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32712744960, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32674201600 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 98 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32748290048, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32712744960 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 99 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32786243584, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32748290048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 100 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32824643584, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32786243584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 101 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32864509952, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32824643584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 102 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32902938624, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32864509952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 103 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32940679168, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32902938624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 104 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:32977575936, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32940679168 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 105 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33016807424, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:32977575936 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 106 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33057738752, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33016807424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 107 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33095532544, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33057738752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 108 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33134252032, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33095532544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 109 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33170997248, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33134252032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 110 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33170997248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 111 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 112 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 113 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 114 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 115 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 116 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 117 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 118 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 119 secs for /mars/resource-lv-2/log-000000001-istore-test-bs7 act = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240, old = /mars/resource-lv-2/log-000000001-istore-test-bs7:33191783240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000001-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000001-istore-test-bs7,33191783240,0 + waited 2 secs for replay to stop. old = log-000000001-istore-test-bs7,33191783240,0, act = log-000000001-istore-test-bs7,33191783240,0 + waited 3 secs for replay to stop. old = log-000000001-istore-test-bs7,33191783240,0, act = log-000000001-istore-test-bs7,33191783240,0 + waited 4 secs for replay to stop. old = log-000000001-istore-test-bs7,33191783240,0, act = log-000000001-istore-test-bs7,33191783240,0 + waited 5 secs for replay to stop. old = log-000000001-istore-test-bs7,33191783240,0, act = log-000000001-istore-test-bs7,33191783240,0 + synctime = 79 + patchlength 500000, no of patches 4 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 7.28069 s, 70.3 MB/s + patching /dev/vg-mars/lv-2 at 524288 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 8.64304 s, 59.2 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 10.0182 s, 51.1 MB/s + patching /dev/vg-mars/lv-2 at 1572864 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 11.5395 s, 44.4 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 30154 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 30154 1 0 13:34 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 30154 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 108359680 + waited 2 secs for syncstatus to stop. old = 108359680, act = 164814848 + waited 3 secs for syncstatus to stop. old = 164814848, act = 196308992 + waited 4 secs for syncstatus to stop. old = 196308992, act = 225595392 + waited 5 secs for syncstatus to stop. old = 225595392, act = 231481344 + waited 6 secs for syncstatus to stop. old = 231481344, act = 258764800 + waited 7 secs for syncstatus to stop. old = 258764800, act = 262078464 + waited 8 secs for syncstatus to stop. old = 262078464, act = 286924800 + waited 9 secs for syncstatus to stop. old = 286924800, act = 298909696 + waited 10 secs for syncstatus to stop. old = 298909696, act = 327684096 + waited 11 secs for syncstatus to stop. old = 327684096, act = 335208448 + waited 12 secs for syncstatus to stop. old = 335208448, act = 359387136 + waited 13 secs for syncstatus to stop. old = 359387136, act = 361840640 + waited 14 secs for syncstatus to stop. old = 361840640, act = 392130560 + waited 15 secs for syncstatus to stop. old = 392130560, act = 400203776 + waited 16 secs for syncstatus to stop. old = 400203776, act = 412991488 + waited 17 secs for syncstatus to stop. old = 412991488, act = 447283200 + waited 18 secs for syncstatus to stop. old = 447283200, act = 461590528 + waited 19 secs for syncstatus to stop. old = 461590528, act = 486248448 + waited 20 secs for syncstatus to stop. old = 486248448, act = 494981120 + waited 21 secs for syncstatus to stop. old = 494981120, act = 618475520 + waited 22 secs for syncstatus to stop. old = 618475520, act = 667029504 + waited 23 secs for syncstatus to stop. old = 667029504, act = 715603968 + waited 24 secs for syncstatus to stop. old = 715603968, act = 743313408 + waited 25 secs for syncstatus to stop. old = 743313408, act = 826851328 + waited 26 secs for syncstatus to stop. old = 826851328, act = 830476288 + waited 27 secs for syncstatus to stop. old = 830476288, act = 830500864 + waited 28 secs for syncstatus to stop. old = 830500864, act = 830500864 + waited 29 secs for syncstatus to stop. old = 830500864, act = 830500864 + waited 30 secs for syncstatus to stop. old = 830500864, act = 830500864 + waited 31 secs for syncstatus to stop. old = 830500864, act = 830500864 + waited 32 secs for syncstatus to stop. old = 830500864, act = 830513152 + waited 33 secs for syncstatus to stop. old = 830513152, act = 845565952 + waited 34 secs for syncstatus to stop. old = 845565952, act = 864067584 + waited 35 secs for syncstatus to stop. old = 864067584, act = 865255424 + waited 36 secs for syncstatus to stop. old = 865255424, act = 933314560 + waited 37 secs for syncstatus to stop. old = 933314560, act = 933314560 + waited 38 secs for syncstatus to stop. old = 933314560, act = 958423040 + waited 39 secs for syncstatus to stop. old = 958423040, act = 966868992 + waited 40 secs for syncstatus to stop. old = 966868992, act = 966950912 + waited 41 secs for syncstatus to stop. old = 966950912, act = 1000423424 + waited 42 secs for syncstatus to stop. old = 1000423424, act = 1041088512 + waited 43 secs for syncstatus to stop. old = 1041088512, act = 1057095680 + waited 44 secs for syncstatus to stop. old = 1057095680, act = 1075847168 + waited 45 secs for syncstatus to stop. old = 1075847168, act = 1101066240 + waited 46 secs for syncstatus to stop. old = 1101066240, act = 1107099648 + waited 47 secs for syncstatus to stop. old = 1107099648, act = 1137270784 + waited 48 secs for syncstatus to stop. old = 1137270784, act = 1144586240 + waited 49 secs for syncstatus to stop. old = 1144586240, act = 1168375808 + waited 50 secs for syncstatus to stop. old = 1168375808, act = 1172901888 + waited 51 secs for syncstatus to stop. old = 1172901888, act = 1182015488 + waited 52 secs for syncstatus to stop. old = 1182015488, act = 1203597312 + waited 53 secs for syncstatus to stop. old = 1203597312, act = 1208823808 + waited 54 secs for syncstatus to stop. old = 1208823808, act = 1233129472 + waited 55 secs for syncstatus to stop. old = 1233129472, act = 1237909504 + waited 56 secs for syncstatus to stop. old = 1237909504, act = 1248366592 + waited 57 secs for syncstatus to stop. old = 1248366592, act = 1268318208 + waited 58 secs for syncstatus to stop. old = 1268318208, act = 1287319552 + waited 59 secs for syncstatus to stop. old = 1287319552, act = 1287319552 + waited 60 secs for syncstatus to stop. old = 1287319552, act = 1287319552 + waited 61 secs for syncstatus to stop. old = 1287319552, act = 1287319552 + waited 62 secs for syncstatus to stop. old = 1287319552, act = 1287319552 + waited 63 secs for syncstatus to stop. old = 1287319552, act = 1320873984 + waited 64 secs for syncstatus to stop. old = 1320873984, act = 1320873984 + waited 65 secs for syncstatus to stop. old = 1320873984, act = 1332215808 + waited 66 secs for syncstatus to stop. old = 1332215808, act = 1364787200 + waited 67 secs for syncstatus to stop. old = 1364787200, act = 1386283008 + waited 68 secs for syncstatus to stop. old = 1386283008, act = 1400848384 + waited 69 secs for syncstatus to stop. old = 1400848384, act = 1417682944 + waited 70 secs for syncstatus to stop. old = 1417682944, act = 1417682944 + waited 71 secs for syncstatus to stop. old = 1417682944, act = 1417682944 + waited 72 secs for syncstatus to stop. old = 1417682944, act = 1451237376 + waited 73 secs for syncstatus to stop. old = 1451237376, act = 1451237376 + waited 74 secs for syncstatus to stop. old = 1451237376, act = 1451237376 + waited 75 secs for syncstatus to stop. old = 1451237376, act = 1484832768 + waited 76 secs for syncstatus to stop. old = 1484832768, act = 1485508608 + waited 77 secs for syncstatus to stop. old = 1485508608, act = 1521852416 + waited 78 secs for syncstatus to stop. old = 1521852416, act = 1537642496 + waited 79 secs for syncstatus to stop. old = 1537642496, act = 1540947968 + waited 80 secs for syncstatus to stop. old = 1540947968, act = 1551900672 + waited 81 secs for syncstatus to stop. old = 1551900672, act = 1551900672 + waited 82 secs for syncstatus to stop. old = 1551900672, act = 1582845952 + waited 83 secs for syncstatus to stop. old = 1582845952, act = 1582845952 + waited 84 secs for syncstatus to stop. old = 1582845952, act = 1585512448 + waited 85 secs for syncstatus to stop. old = 1585512448, act = 1605296128 + waited 86 secs for syncstatus to stop. old = 1605296128, act = 1619009536 + waited 87 secs for syncstatus to stop. old = 1619009536, act = 1624297472 + waited 88 secs for syncstatus to stop. old = 1624297472, act = 1666113536 + waited 89 secs for syncstatus to stop. old = 1666113536, act = 1666113536 + waited 90 secs for syncstatus to stop. old = 1666113536, act = 1666113536 + waited 91 secs for syncstatus to stop. old = 1666113536, act = 1666113536 + waited 92 secs for syncstatus to stop. old = 1666113536, act = 1666113536 + waited 93 secs for syncstatus to stop. old = 1666113536, act = 1666113536 + waited 94 secs for syncstatus to stop. old = 1666113536, act = 1686114304 + waited 95 secs for syncstatus to stop. old = 1686114304, act = 1690370048 + waited 96 secs for syncstatus to stop. old = 1690370048, act = 1719668736 + waited 97 secs for syncstatus to stop. old = 1719668736, act = 1719668736 + waited 98 secs for syncstatus to stop. old = 1719668736, act = 1719668736 + waited 99 secs for syncstatus to stop. old = 1719668736, act = 1726615552 + waited 100 secs for syncstatus to stop. old = 1726615552, act = 1753960448 + waited 101 secs for syncstatus to stop. old = 1753960448, act = 1772736512 + waited 102 secs for syncstatus to stop. old = 1772736512, act = 1779286016 + waited 103 secs for syncstatus to stop. old = 1779286016, act = 1793814528 + waited 104 secs for syncstatus to stop. old = 1793814528, act = 1805955072 + waited 105 secs for syncstatus to stop. old = 1805955072, act = 1820217344 + waited 106 secs for syncstatus to stop. old = 1820217344, act = 1831956480 + waited 107 secs for syncstatus to stop. old = 1831956480, act = 1842094080 + waited 108 secs for syncstatus to stop. old = 1842094080, act = 1842094080 + waited 109 secs for syncstatus to stop. old = 1842094080, act = 1848180736 + waited 110 secs for syncstatus to stop. old = 1848180736, act = 1848180736 + waited 111 secs for syncstatus to stop. old = 1848180736, act = 1870180352 + waited 112 secs for syncstatus to stop. old = 1870180352, act = 1881735168 + waited 113 secs for syncstatus to stop. old = 1881735168, act = 1881735168 + waited 114 secs for syncstatus to stop. old = 1881735168, act = 1897766912 + waited 115 secs for syncstatus to stop. old = 1897766912, act = 1915289600 + waited 116 secs for syncstatus to stop. old = 1915289600, act = 1915289600 + waited 117 secs for syncstatus to stop. old = 1915289600, act = 1915289600 + waited 118 secs for syncstatus to stop. old = 1915289600, act = 1948495872 + waited 119 secs for syncstatus to stop. old = 1948495872, act = 1948844032 + waited 120 secs for syncstatus to stop. old = 1948844032, act = 1948844032 + waited 121 secs for syncstatus to stop. old = 1948844032, act = 1948844032 + waited 122 secs for syncstatus to stop. old = 1948844032, act = 1948844032 + waited 123 secs for syncstatus to stop. old = 1948844032, act = 1948844032 + waited 124 secs for syncstatus to stop. old = 1948844032, act = 1948844032 + waited 125 secs for syncstatus to stop. old = 1948844032, act = 1982398464 + waited 126 secs for syncstatus to stop. old = 1982398464, act = 1983578112 + waited 127 secs for syncstatus to stop. old = 1983578112, act = 2011136000 + waited 128 secs for syncstatus to stop. old = 2011136000, act = 2011136000 + waited 129 secs for syncstatus to stop. old = 2011136000, act = 2017349632 + waited 130 secs for syncstatus to stop. old = 2017349632, act = 2049507328 + waited 131 secs for syncstatus to stop. old = 2049507328, act = 2049531904 + waited 132 secs for syncstatus to stop. old = 2049531904, act = 2073141248 + waited 133 secs for syncstatus to stop. old = 2073141248, act = 2083061760 + waited 134 secs for syncstatus to stop. old = 2083061760, act = 2119475200 + waited 135 secs for syncstatus to stop. old = 2119475200, act = 2126864384 + waited 136 secs for syncstatus to stop. old = 2126864384, act = 2131595264 + waited 137 secs for syncstatus to stop. old = 2131595264, act = 2140639232 + waited 138 secs for syncstatus to stop. old = 2140639232, act = 2144620544 + waited 139 secs for syncstatus to stop. old = 2144620544, act = 2147483648 + waited 140 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 141 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 142 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 143 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 144 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 145 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 146 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 147 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 148 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 149 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 150 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 151 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 152 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 153 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 154 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 155 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 156 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 157 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 158 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=30154) on istore-test-bs7 + write_count: 46 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:46, time_sync=572, rate = 4 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1250967552, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1289502720, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1250967552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1331113984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1289502720 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1370189824, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1331113984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1410199552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1370189824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1449750528, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1410199552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1487441920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1449750528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1524129792, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1487441920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1561628672, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1524129792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1602129920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1561628672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1642717184, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1602129920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1678495744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1642717184 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1715765248, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1678495744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1756352512, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1715765248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1793282048, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1756352512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1799323648, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1793282048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1806127104, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1799323648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1811640320, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1806127104 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1815629824, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1811640320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1819344896, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1815629824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1827069952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1819344896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1867612160, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1827069952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1905655808, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1867612160 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1951711232, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1905655808 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:1973985280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1951711232 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2011570176, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:1973985280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2049466368, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2011570176 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2085220352, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2049466368 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2118328320, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2085220352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2151276544, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2118328320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2186862592, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2151276544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2224656384, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2186862592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2261340160, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2224656384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2301427712, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2261340160 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2338422784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2301427712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2376355840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2338422784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2414743552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2376355840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2452566016, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2414743552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2489319424, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2452566016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2527096832, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2489319424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2565271552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2527096832 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2602098688, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2565271552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2638659584, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2602098688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2675032064, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2638659584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2714873856, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2675032064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2754715648, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2714873856 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2790277120, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2754715648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2828824576, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2790277120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2865635328, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2828824576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2901696512, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2865635328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2942603264, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2901696512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:2980376576, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2942603264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3017519104, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:2980376576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3050475520, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3017519104 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3089059840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3050475520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3126693888, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3089059840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3163475968, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3126693888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3205947392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3163475968 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3245375488, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3205947392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3285815296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3245375488 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3324264448, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3285815296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3364335616, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3324264448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3401428992, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3364335616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3435040768, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3401428992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3471028224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3435040768 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3505143808, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3471028224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3541807104, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3505143808 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3579822080, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3541807104 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3618045952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3579822080 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3657904128, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3618045952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3693748224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3657904128 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3733909504, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3693748224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3773751296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3733909504 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3813392384, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3773751296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3849478144, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3813392384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3885223936, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3849478144 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3924971520, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3885223936 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3960545280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3924971520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:3994624000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3960545280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4035461120, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:3994624000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4073111552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4035461120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4111437824, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4073111552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4149526528, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4111437824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4190011392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4149526528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4225417216, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4190011392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4264591360, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4225417216 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4305240064, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4264591360 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 88 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4403056640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4305240064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 89 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4438462464, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4403056640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 90 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4478124032, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4438462464 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 91 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4513886208, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4478124032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 92 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4553027584, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4513886208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 93 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4588290048, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4553027584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 94 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4626247680, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4588290048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 95 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4668235776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4626247680 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 96 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4706476032, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4668235776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 97 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4743753728, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4706476032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 98 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4779872256, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4743753728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 99 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4816367616, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4779872256 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 100 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4855336960, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4816367616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 101 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4893974528, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4855336960 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 102 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4928774144, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4893974528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 103 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:4965371904, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4928774144 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 104 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5004783616, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:4965371904 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 105 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5043716096, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5004783616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 106 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5080956928, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5043716096 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 107 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5115039744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5080956928 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 108 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5150683136, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5115039744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 109 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5191335936, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5150683136 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 110 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5227524096, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5191335936 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 111 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5266014208, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5227524096 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 112 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5303328768, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5266014208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 113 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5343432704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5303328768 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 114 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5343432704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 115 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 116 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 117 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 118 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 119 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 120 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 121 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 122 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 123 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:5371500640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000002-istore-test-bs7,5371500640,0 + waited 2 secs for replay to stop. old = log-000000002-istore-test-bs7,5371500640,0, act = log-000000002-istore-test-bs7,5371500640,0 + waited 3 secs for replay to stop. old = log-000000002-istore-test-bs7,5371500640,0, act = log-000000002-istore-test-bs7,5371500640,0 + waited 4 secs for replay to stop. old = log-000000002-istore-test-bs7,5371500640,0, act = log-000000002-istore-test-bs7,5371500640,0 + waited 5 secs for replay to stop. old = log-000000002-istore-test-bs7,5371500640,0, act = log-000000002-istore-test-bs7,5371500640,0 + synctime = 138 + patchlength 1000000, no of patches 2 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 14.6272 s, 70.0 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 17.1973 s, 59.5 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 2003 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 2003 1 0 13:48 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 2003 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 115027968 + waited 2 secs for syncstatus to stop. old = 115027968, act = 143855616 + waited 3 secs for syncstatus to stop. old = 143855616, act = 234000384 + waited 4 secs for syncstatus to stop. old = 234000384, act = 234000384 + waited 5 secs for syncstatus to stop. old = 234000384, act = 288333824 + waited 6 secs for syncstatus to stop. old = 288333824, act = 288333824 + waited 7 secs for syncstatus to stop. old = 288333824, act = 350212096 + waited 8 secs for syncstatus to stop. old = 350212096, act = 379949056 + waited 9 secs for syncstatus to stop. old = 379949056, act = 395137024 + waited 10 secs for syncstatus to stop. old = 395137024, act = 408752128 + waited 11 secs for syncstatus to stop. old = 408752128, act = 413118464 + waited 12 secs for syncstatus to stop. old = 413118464, act = 435867648 + waited 13 secs for syncstatus to stop. old = 435867648, act = 442834944 + waited 14 secs for syncstatus to stop. old = 442834944, act = 464400384 + waited 15 secs for syncstatus to stop. old = 464400384, act = 475860992 + waited 16 secs for syncstatus to stop. old = 475860992, act = 507699200 + waited 17 secs for syncstatus to stop. old = 507699200, act = 520015872 + waited 18 secs for syncstatus to stop. old = 520015872, act = 594591744 + waited 19 secs for syncstatus to stop. old = 594591744, act = 605765632 + waited 20 secs for syncstatus to stop. old = 605765632, act = 621318144 + waited 21 secs for syncstatus to stop. old = 621318144, act = 621318144 + waited 22 secs for syncstatus to stop. old = 621318144, act = 662904832 + waited 23 secs for syncstatus to stop. old = 662904832, act = 670777344 + waited 24 secs for syncstatus to stop. old = 670777344, act = 673861632 + waited 25 secs for syncstatus to stop. old = 673861632, act = 701976576 + waited 26 secs for syncstatus to stop. old = 701976576, act = 707014656 + waited 27 secs for syncstatus to stop. old = 707014656, act = 722587648 + waited 28 secs for syncstatus to stop. old = 722587648, act = 722587648 + waited 29 secs for syncstatus to stop. old = 722587648, act = 722587648 + waited 30 secs for syncstatus to stop. old = 722587648, act = 722587648 + waited 31 secs for syncstatus to stop. old = 722587648, act = 722587648 + waited 32 secs for syncstatus to stop. old = 722587648, act = 732385280 + waited 33 secs for syncstatus to stop. old = 732385280, act = 757137408 + waited 34 secs for syncstatus to stop. old = 757137408, act = 779456512 + waited 35 secs for syncstatus to stop. old = 779456512, act = 791740416 + waited 36 secs for syncstatus to stop. old = 791740416, act = 794902528 + waited 37 secs for syncstatus to stop. old = 794902528, act = 828633088 + waited 38 secs for syncstatus to stop. old = 828633088, act = 828633088 + waited 39 secs for syncstatus to stop. old = 828633088, act = 864137216 + waited 40 secs for syncstatus to stop. old = 864137216, act = 881983488 + waited 41 secs for syncstatus to stop. old = 881983488, act = 895614976 + waited 42 secs for syncstatus to stop. old = 895614976, act = 904560640 + waited 43 secs for syncstatus to stop. old = 904560640, act = 919740416 + waited 44 secs for syncstatus to stop. old = 919740416, act = 926003200 + waited 45 secs for syncstatus to stop. old = 926003200, act = 980983808 + waited 46 secs for syncstatus to stop. old = 980983808, act = 1032126464 + waited 47 secs for syncstatus to stop. old = 1032126464, act = 1079824384 + waited 48 secs for syncstatus to stop. old = 1079824384, act = 1141690368 + waited 49 secs for syncstatus to stop. old = 1141690368, act = 1186529280 + waited 50 secs for syncstatus to stop. old = 1186529280, act = 1208487936 + waited 51 secs for syncstatus to stop. old = 1208487936, act = 1208487936 + waited 52 secs for syncstatus to stop. old = 1208487936, act = 1265504256 + waited 53 secs for syncstatus to stop. old = 1265504256, act = 1274085376 + waited 54 secs for syncstatus to stop. old = 1274085376, act = 1283076096 + waited 55 secs for syncstatus to stop. old = 1283076096, act = 1318514688 + waited 56 secs for syncstatus to stop. old = 1318514688, act = 1354563584 + waited 57 secs for syncstatus to stop. old = 1354563584, act = 1357709312 + waited 58 secs for syncstatus to stop. old = 1357709312, act = 1391546368 + waited 59 secs for syncstatus to stop. old = 1391546368, act = 1406578688 + waited 60 secs for syncstatus to stop. old = 1406578688, act = 1425231872 + waited 61 secs for syncstatus to stop. old = 1425231872, act = 1425231872 + waited 62 secs for syncstatus to stop. old = 1425231872, act = 1425231872 + waited 63 secs for syncstatus to stop. old = 1425231872, act = 1425231872 + waited 64 secs for syncstatus to stop. old = 1425231872, act = 1426935808 + waited 65 secs for syncstatus to stop. old = 1426935808, act = 1471217664 + waited 66 secs for syncstatus to stop. old = 1471217664, act = 1478828032 + waited 67 secs for syncstatus to stop. old = 1478828032, act = 1489260544 + waited 68 secs for syncstatus to stop. old = 1489260544, act = 1565904896 + waited 69 secs for syncstatus to stop. old = 1565904896, act = 1604370432 + waited 70 secs for syncstatus to stop. old = 1604370432, act = 1650786304 + waited 71 secs for syncstatus to stop. old = 1650786304, act = 1727950848 + waited 72 secs for syncstatus to stop. old = 1727950848, act = 1751687168 + waited 73 secs for syncstatus to stop. old = 1751687168, act = 1764196352 + waited 74 secs for syncstatus to stop. old = 1764196352, act = 1774747648 + waited 75 secs for syncstatus to stop. old = 1774747648, act = 1792581632 + waited 76 secs for syncstatus to stop. old = 1792581632, act = 1816698880 + waited 77 secs for syncstatus to stop. old = 1816698880, act = 1828225024 + waited 78 secs for syncstatus to stop. old = 1828225024, act = 1870827520 + waited 79 secs for syncstatus to stop. old = 1870827520, act = 1874337792 + waited 80 secs for syncstatus to stop. old = 1874337792, act = 1929117696 + waited 81 secs for syncstatus to stop. old = 1929117696, act = 1978204160 + waited 82 secs for syncstatus to stop. old = 1978204160, act = 2043854848 + waited 83 secs for syncstatus to stop. old = 2043854848, act = 2099826688 + waited 84 secs for syncstatus to stop. old = 2099826688, act = 2112139264 + waited 85 secs for syncstatus to stop. old = 2112139264, act = 2138845184 + waited 86 secs for syncstatus to stop. old = 2138845184, act = 2147483648 + waited 87 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 88 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 89 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 90 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 91 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 92 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 93 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 94 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 95 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 96 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 97 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 98 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 99 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 100 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 101 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 102 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 103 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 104 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 105 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=2003) on istore-test-bs7 + write_count: 51 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:51, time_sync=364, rate = 8 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:7884247040, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:7925436416, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:7884247040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:7961579520, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:7925436416 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:7998091264, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:7961579520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8036188160, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:7998091264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8070053888, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8036188160 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8106868736, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8070053888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8142188544, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8106868736 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8179703808, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8142188544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8217636864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8179703808 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8251645952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8217636864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8288137216, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8251645952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8327876608, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8288137216 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8366747648, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8327876608 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8402153472, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8366747648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8440762368, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8402153472 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8478908416, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8440762368 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8517701632, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8478908416 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8555274240, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8517701632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8593813504, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8555274240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8629268480, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8593813504 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8663285760, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8629268480 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8702996480, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8663285760 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8740900864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8702996480 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8777768960, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8740900864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8812535808, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8777768960 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8849907712, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8812535808 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8885575680, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8849907712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8923840512, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8885575680 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:8963850240, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8923840512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9001377792, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:8963850240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9039405056, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9001377792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9076445184, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9039405056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9114451968, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9076445184 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9147232256, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9114451968 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9185775616, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9147232256 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9219526656, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9185775616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9253642240, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9219526656 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9290989568, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9253642240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9329586176, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9290989568 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9364250624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9329586176 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9400836096, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9364250624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9436438528, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9400836096 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9473716224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9436438528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9508995072, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9473716224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9546362880, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9508995072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9582280704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9546362880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9616465920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9582280704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9649737728, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9616465920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9687015424, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9649737728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9723965440, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9687015424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9761300480, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9723965440 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9795211264, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9761300480 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9834606592, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9795211264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9874018304, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9834606592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9912451072, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9874018304 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9947955200, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9912451072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:9987059712, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9947955200 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10027847680, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:9987059712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10063958016, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10027847680 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10100252672, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10063958016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10140037120, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10100252672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10179702784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10140037120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10214756352, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10179702784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10250354688, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10214756352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10287915008, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10250354688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10327617536, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10287915008 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10362880000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10327617536 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10401669120, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10362880000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10441469952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10401669120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10481098752, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10441469952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10516545536, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10481098752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10552926208, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10516545536 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10590089216, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10552926208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10629312512, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10590089216 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10667548672, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10629312512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10702135296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10667548672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10738663424, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10702135296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10776735744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10738663424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10813751296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10776735744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10852093952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10813751296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10891001856, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10852093952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10928848896, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10891001856 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:10965762048, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10928848896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11003576320, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:10965762048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11043074048, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11003576320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11083206656, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11043074048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 88 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11121565696, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11083206656 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 89 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11160920064, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11121565696 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 90 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11199021056, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11160920064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 91 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11240435712, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11199021056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 92 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11276111872, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11240435712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 93 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11314610176, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11276111872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 94 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11351797760, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11314610176 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 95 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11391848448, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11351797760 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 96 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11425685504, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11391848448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 97 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11461238784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11425685504 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 98 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11489056040, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11461238784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 99 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11529375744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11489056040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 100 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11566252032, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11529375744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 101 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11602980864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11566252032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 102 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11640561664, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11602980864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 103 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11677409280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11640561664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 104 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11714326528, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11677409280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 105 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11750916096, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11714326528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 106 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11785752576, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11750916096 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 107 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11819999232, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11785752576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 108 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11857305600, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11819999232 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 109 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11895304192, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11857305600 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 110 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11933294592, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11895304192 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 111 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11933294592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 112 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 113 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 114 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 115 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 116 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 117 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 118 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 119 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 120 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:11961960340 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000002-istore-test-bs7,11961960340,0 + waited 2 secs for replay to stop. old = log-000000002-istore-test-bs7,11961960340,0, act = log-000000002-istore-test-bs7,11961960340,0 + waited 3 secs for replay to stop. old = log-000000002-istore-test-bs7,11961960340,0, act = log-000000002-istore-test-bs7,11961960340,0 + waited 4 secs for replay to stop. old = log-000000002-istore-test-bs7,11961960340,0, act = log-000000002-istore-test-bs7,11961960340,0 + waited 5 secs for replay to stop. old = log-000000002-istore-test-bs7,11961960340,0, act = log-000000002-istore-test-bs7,11961960340,0 + synctime = 85 +use_mars_sync=1 synctimes = 76 79 138 85 +==> Thu Jul 18 13:59:17 CEST 2013 Finished. +=============================================================== + +Ignoring inner directory ./no_fast_sync + +=============================================================== +======== ./no_fast_sync/no_parallel_writer +Sourcing config file /home/fl/mars/test_suite/user_modules.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/no_fast_sync.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/no_parallel_writer.conf +==> Thu Jul 18 13:59:17 CEST 2013 Starting .no_fast_sync.no_parallel_writer +calling synctest_run + setting fast sync mode to 0 on istore-test-bs7 + setting fast sync mode to 0 on istore-test-bap7 + patchlength 1, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.000455517 s, 2.2 MB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.789932 s, 1.3 kB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 1.49639 s, 0.7 kB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.08806 s, 0.5 kB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.78695 s, 0.4 kB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.1788 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.78729 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 4.45957 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 5.29453 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 18.259 s, 0.1 kB/s + patching /dev/vg-mars/lv-2 at 2097150 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 12.9425 s, 0.1 kB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 0 + waited 2 secs for syncstatus to stop. old = 0, act = 0 + waited 3 secs for syncstatus to stop. old = 0, act = 0 + waited 4 secs for syncstatus to stop. old = 0, act = 0 + waited 5 secs for syncstatus to stop. old = 0, act = 3887104 + waited 6 secs for syncstatus to stop. old = 3887104, act = 3887104 + waited 7 secs for syncstatus to stop. old = 3887104, act = 3887104 + waited 8 secs for syncstatus to stop. old = 3887104, act = 3887104 + waited 9 secs for syncstatus to stop. old = 3887104, act = 3887104 + waited 10 secs for syncstatus to stop. old = 3887104, act = 3887104 + waited 11 secs for syncstatus to stop. old = 3887104, act = 3887104 + waited 12 secs for syncstatus to stop. old = 3887104, act = 20897792 + waited 13 secs for syncstatus to stop. old = 20897792, act = 37076992 + waited 14 secs for syncstatus to stop. old = 37076992, act = 88182784 + waited 15 secs for syncstatus to stop. old = 88182784, act = 152645632 + waited 16 secs for syncstatus to stop. old = 152645632, act = 205430784 + waited 17 secs for syncstatus to stop. old = 205430784, act = 247435264 + waited 18 secs for syncstatus to stop. old = 247435264, act = 314859520 + waited 19 secs for syncstatus to stop. old = 314859520, act = 373788672 + waited 20 secs for syncstatus to stop. old = 373788672, act = 415551488 + waited 21 secs for syncstatus to stop. old = 415551488, act = 460394496 + waited 22 secs for syncstatus to stop. old = 460394496, act = 527421440 + waited 23 secs for syncstatus to stop. old = 527421440, act = 585998336 + waited 24 secs for syncstatus to stop. old = 585998336, act = 625393664 + waited 25 secs for syncstatus to stop. old = 625393664, act = 666685440 + waited 26 secs for syncstatus to stop. old = 666685440, act = 732934144 + waited 27 secs for syncstatus to stop. old = 732934144, act = 749056000 + waited 28 secs for syncstatus to stop. old = 749056000, act = 847859712 + waited 29 secs for syncstatus to stop. old = 847859712, act = 890847232 + waited 30 secs for syncstatus to stop. old = 890847232, act = 952336384 + waited 31 secs for syncstatus to stop. old = 952336384, act = 995016704 + waited 32 secs for syncstatus to stop. old = 995016704, act = 1051865088 + waited 33 secs for syncstatus to stop. old = 1051865088, act = 1110028288 + waited 34 secs for syncstatus to stop. old = 1110028288, act = 1153150976 + waited 35 secs for syncstatus to stop. old = 1153150976, act = 1217191936 + waited 36 secs for syncstatus to stop. old = 1217191936, act = 1275060224 + waited 37 secs for syncstatus to stop. old = 1275060224, act = 1312620544 + waited 38 secs for syncstatus to stop. old = 1312620544, act = 1415208960 + waited 39 secs for syncstatus to stop. old = 1415208960, act = 1533444096 + waited 40 secs for syncstatus to stop. old = 1533444096, act = 1621901312 + waited 41 secs for syncstatus to stop. old = 1621901312, act = 1731379200 + waited 42 secs for syncstatus to stop. old = 1731379200, act = 1789870080 + waited 43 secs for syncstatus to stop. old = 1789870080, act = 1832505344 + waited 44 secs for syncstatus to stop. old = 1832505344, act = 1874350080 + waited 45 secs for syncstatus to stop. old = 1874350080, act = 1937846272 + waited 46 secs for syncstatus to stop. old = 1937846272, act = 1996156928 + waited 47 secs for syncstatus to stop. old = 1996156928, act = 2056577024 + waited 48 secs for syncstatus to stop. old = 2056577024, act = 2097377280 + waited 49 secs for syncstatus to stop. old = 2097377280, act = 2147483648 + waited 50 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 51 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 52 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 53 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 54 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 55 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 56 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 57 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 58 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 59 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 60 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 61 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 62 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 63 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 64 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 65 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 66 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 67 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 68 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + synctime = 48 + patchlength 50000, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 0.745252 s, 68.7 MB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 1.44271 s, 35.5 MB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.19923 s, 23.3 MB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.7944 s, 18.3 MB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.44493 s, 14.9 MB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.88816 s, 13.2 MB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 4.48141 s, 11.4 MB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.12581 s, 10.0 MB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.69398 s, 9.0 MB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 6.83049 s, 7.5 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 158261248 + waited 2 secs for syncstatus to stop. old = 158261248, act = 201281536 + waited 3 secs for syncstatus to stop. old = 201281536, act = 264986624 + waited 4 secs for syncstatus to stop. old = 264986624, act = 320798720 + waited 5 secs for syncstatus to stop. old = 320798720, act = 363819008 + waited 6 secs for syncstatus to stop. old = 363819008, act = 405176320 + waited 7 secs for syncstatus to stop. old = 405176320, act = 461058048 + waited 8 secs for syncstatus to stop. old = 461058048, act = 518119424 + waited 9 secs for syncstatus to stop. old = 518119424, act = 556806144 + waited 10 secs for syncstatus to stop. old = 556806144, act = 621281280 + waited 11 secs for syncstatus to stop. old = 621281280, act = 662335488 + waited 12 secs for syncstatus to stop. old = 662335488, act = 722370560 + waited 13 secs for syncstatus to stop. old = 722370560, act = 765923328 + waited 14 secs for syncstatus to stop. old = 765923328, act = 832258048 + waited 15 secs for syncstatus to stop. old = 832258048, act = 888545280 + waited 16 secs for syncstatus to stop. old = 888545280, act = 931373056 + waited 17 secs for syncstatus to stop. old = 931373056, act = 974737408 + waited 18 secs for syncstatus to stop. old = 974737408, act = 1042198528 + waited 19 secs for syncstatus to stop. old = 1042198528, act = 1094955008 + waited 20 secs for syncstatus to stop. old = 1094955008, act = 1137131520 + waited 21 secs for syncstatus to stop. old = 1137131520, act = 1201684480 + waited 22 secs for syncstatus to stop. old = 1201684480, act = 1266417664 + waited 23 secs for syncstatus to stop. old = 1266417664, act = 1291153408 + waited 24 secs for syncstatus to stop. old = 1291153408, act = 1305759744 + waited 25 secs for syncstatus to stop. old = 1305759744, act = 1320062976 + waited 26 secs for syncstatus to stop. old = 1320062976, act = 1403043840 + waited 27 secs for syncstatus to stop. old = 1403043840, act = 1439662080 + waited 28 secs for syncstatus to stop. old = 1439662080, act = 1451696128 + waited 29 secs for syncstatus to stop. old = 1451696128, act = 1515552768 + waited 30 secs for syncstatus to stop. old = 1515552768, act = 1545609216 + waited 31 secs for syncstatus to stop. old = 1545609216, act = 1608884224 + waited 32 secs for syncstatus to stop. old = 1608884224, act = 1664057344 + waited 33 secs for syncstatus to stop. old = 1664057344, act = 1699782656 + waited 34 secs for syncstatus to stop. old = 1699782656, act = 1763184640 + waited 35 secs for syncstatus to stop. old = 1763184640, act = 1810751488 + waited 36 secs for syncstatus to stop. old = 1810751488, act = 1838256128 + waited 37 secs for syncstatus to stop. old = 1838256128, act = 1896980480 + waited 38 secs for syncstatus to stop. old = 1896980480, act = 1956614144 + waited 39 secs for syncstatus to stop. old = 1956614144, act = 1995759616 + waited 40 secs for syncstatus to stop. old = 1995759616, act = 2063351808 + waited 41 secs for syncstatus to stop. old = 2063351808, act = 2102992896 + waited 42 secs for syncstatus to stop. old = 2102992896, act = 2147483648 + waited 43 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 44 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 45 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 46 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 47 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 48 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 49 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 50 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 51 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 52 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 53 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 54 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 55 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 56 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 57 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 58 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 59 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 60 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 61 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + synctime = 41 + patchlength 500000, no of patches 4 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 7.36217 s, 69.5 MB/s + patching /dev/vg-mars/lv-2 at 524288 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 8.69477 s, 58.9 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 10.0965 s, 50.7 MB/s + patching /dev/vg-mars/lv-2 at 1572864 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 11.9337 s, 42.9 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 122724352 + waited 2 secs for syncstatus to stop. old = 122724352, act = 176504832 + waited 3 secs for syncstatus to stop. old = 176504832, act = 216399872 + waited 4 secs for syncstatus to stop. old = 216399872, act = 282816512 + waited 5 secs for syncstatus to stop. old = 282816512, act = 341860352 + waited 6 secs for syncstatus to stop. old = 341860352, act = 385597440 + waited 7 secs for syncstatus to stop. old = 385597440, act = 424034304 + waited 8 secs for syncstatus to stop. old = 424034304, act = 490364928 + waited 9 secs for syncstatus to stop. old = 490364928, act = 550043648 + waited 10 secs for syncstatus to stop. old = 550043648, act = 604168192 + waited 11 secs for syncstatus to stop. old = 604168192, act = 657551360 + waited 12 secs for syncstatus to stop. old = 657551360, act = 702382080 + waited 13 secs for syncstatus to stop. old = 702382080, act = 762155008 + waited 14 secs for syncstatus to stop. old = 762155008, act = 819273728 + waited 15 secs for syncstatus to stop. old = 819273728, act = 857456640 + waited 16 secs for syncstatus to stop. old = 857456640, act = 918216704 + waited 17 secs for syncstatus to stop. old = 918216704, act = 974778368 + waited 18 secs for syncstatus to stop. old = 974778368, act = 1018544128 + waited 19 secs for syncstatus to stop. old = 1018544128, act = 1076289536 + waited 20 secs for syncstatus to stop. old = 1076289536, act = 1131941888 + waited 21 secs for syncstatus to stop. old = 1131941888, act = 1170296832 + waited 22 secs for syncstatus to stop. old = 1170296832, act = 1237426176 + waited 23 secs for syncstatus to stop. old = 1237426176, act = 1288294400 + waited 24 secs for syncstatus to stop. old = 1288294400, act = 1331572736 + waited 25 secs for syncstatus to stop. old = 1331572736, act = 1373118464 + waited 26 secs for syncstatus to stop. old = 1373118464, act = 1443184640 + waited 27 secs for syncstatus to stop. old = 1443184640, act = 1494278144 + waited 28 secs for syncstatus to stop. old = 1494278144, act = 1510338560 + waited 29 secs for syncstatus to stop. old = 1510338560, act = 1573486592 + waited 30 secs for syncstatus to stop. old = 1573486592, act = 1610235904 + waited 31 secs for syncstatus to stop. old = 1610235904, act = 1678950400 + waited 32 secs for syncstatus to stop. old = 1678950400, act = 1730039808 + waited 33 secs for syncstatus to stop. old = 1730039808, act = 1772634112 + waited 34 secs for syncstatus to stop. old = 1772634112, act = 1829191680 + waited 35 secs for syncstatus to stop. old = 1829191680, act = 1894834176 + waited 36 secs for syncstatus to stop. old = 1894834176, act = 1918414848 + waited 37 secs for syncstatus to stop. old = 1918414848, act = 1962512384 + waited 38 secs for syncstatus to stop. old = 1962512384, act = 2042028032 + waited 39 secs for syncstatus to stop. old = 2042028032, act = 2102820864 + waited 40 secs for syncstatus to stop. old = 2102820864, act = 2147483648 + waited 41 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 42 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 43 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 44 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 45 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 46 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 47 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 48 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 49 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 50 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 51 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 52 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 53 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 54 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 55 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 56 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 57 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 58 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 59 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + synctime = 39 + patchlength 1000000, no of patches 2 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 14.6828 s, 69.7 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 17.0673 s, 60.0 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 116957184 + waited 2 secs for syncstatus to stop. old = 116957184, act = 157024256 + waited 3 secs for syncstatus to stop. old = 157024256, act = 212070400 + waited 4 secs for syncstatus to stop. old = 212070400, act = 263987200 + waited 5 secs for syncstatus to stop. old = 263987200, act = 314486784 + waited 6 secs for syncstatus to stop. old = 314486784, act = 367755264 + waited 7 secs for syncstatus to stop. old = 367755264, act = 417800192 + waited 8 secs for syncstatus to stop. old = 417800192, act = 458694656 + waited 9 secs for syncstatus to stop. old = 458694656, act = 505794560 + waited 10 secs for syncstatus to stop. old = 505794560, act = 543969280 + waited 11 secs for syncstatus to stop. old = 543969280, act = 599629824 + waited 12 secs for syncstatus to stop. old = 599629824, act = 652517376 + waited 13 secs for syncstatus to stop. old = 652517376, act = 716656640 + waited 14 secs for syncstatus to stop. old = 716656640, act = 767365120 + waited 15 secs for syncstatus to stop. old = 767365120, act = 808140800 + waited 16 secs for syncstatus to stop. old = 808140800, act = 879976448 + waited 17 secs for syncstatus to stop. old = 879976448, act = 931414016 + waited 18 secs for syncstatus to stop. old = 931414016, act = 974118912 + waited 19 secs for syncstatus to stop. old = 974118912, act = 1018372096 + waited 20 secs for syncstatus to stop. old = 1018372096, act = 1086738432 + waited 21 secs for syncstatus to stop. old = 1086738432, act = 1123770368 + waited 22 secs for syncstatus to stop. old = 1123770368, act = 1182031872 + waited 23 secs for syncstatus to stop. old = 1182031872, act = 1224171520 + waited 24 secs for syncstatus to stop. old = 1224171520, act = 1278349312 + waited 25 secs for syncstatus to stop. old = 1278349312, act = 1348476928 + waited 26 secs for syncstatus to stop. old = 1348476928, act = 1386561536 + waited 27 secs for syncstatus to stop. old = 1386561536, act = 1443848192 + waited 28 secs for syncstatus to stop. old = 1443848192, act = 1443848192 + waited 29 secs for syncstatus to stop. old = 1443848192, act = 1548103680 + waited 30 secs for syncstatus to stop. old = 1548103680, act = 1670705152 + waited 31 secs for syncstatus to stop. old = 1670705152, act = 1709826048 + waited 32 secs for syncstatus to stop. old = 1709826048, act = 1769562112 + waited 33 secs for syncstatus to stop. old = 1769562112, act = 1814822912 + waited 34 secs for syncstatus to stop. old = 1814822912, act = 1842569216 + waited 35 secs for syncstatus to stop. old = 1842569216, act = 1900998656 + waited 36 secs for syncstatus to stop. old = 1900998656, act = 1941872640 + waited 37 secs for syncstatus to stop. old = 1941872640, act = 1977503744 + waited 38 secs for syncstatus to stop. old = 1977503744, act = 2039316480 + waited 39 secs for syncstatus to stop. old = 2039316480, act = 2089353216 + waited 40 secs for syncstatus to stop. old = 2089353216, act = 2129235968 + waited 41 secs for syncstatus to stop. old = 2129235968, act = 2147483648 + waited 42 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 43 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 44 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 45 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 46 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 47 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 48 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 49 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 50 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 51 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 52 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 53 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 54 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 55 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 56 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 57 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 58 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 59 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 60 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bs7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + calculating cksum for /dev/vg-mars/lv-2 on istore-test-bap7 + cksum = 2529392344 2147483648 /dev/vg-mars/lv-2 + synctime = 40 +use_mars_sync=1 synctimes = 48 41 39 40 +==> Thu Jul 18 14:17:29 CEST 2013 Finished. +=============================================================== + + +=============================================================== +======== ./no_fast_sync/parallel_writer +Sourcing config file /home/fl/mars/test_suite/user_modules.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/no_fast_sync.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/parallel_writer.conf +==> Thu Jul 18 14:17:29 CEST 2013 Starting .no_fast_sync.parallel_writer +calling synctest_run + setting fast sync mode to 0 on istore-test-bs7 + setting fast sync mode to 0 on istore-test-bap7 + checking whether /dev/mars/lv-2 is mounted on istore-test-bs7 +/dev/mars/lv-2 on /mnt/test type ext3 (rw) + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 10373 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 10373 1 0 14:17 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 10373 has errors in /tmp/write_and_delete.sh.13181.err + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=10373) on istore-test-bs7 + write_count: 8 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:8, time_write=20, rate = 24 / min + patchlength 1, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.00820345 s, 125 kB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.762731 s, 1.3 kB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 1.43386 s, 0.7 kB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.16619 s, 0.5 kB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.87902 s, 0.4 kB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.29459 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.97373 s, 0.3 kB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 4.55068 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 5.33317 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 6.06316 s, 0.2 kB/s + patching /dev/vg-mars/lv-2 at 2097150 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 6.5098 s, 0.2 kB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 11771 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 11771 1 0 14:19 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 11771 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 140541952 + waited 2 secs for syncstatus to stop. old = 140541952, act = 201093120 + waited 3 secs for syncstatus to stop. old = 201093120, act = 248897536 + waited 4 secs for syncstatus to stop. old = 248897536, act = 277676032 + waited 5 secs for syncstatus to stop. old = 277676032, act = 310550528 + waited 6 secs for syncstatus to stop. old = 310550528, act = 347619328 + waited 7 secs for syncstatus to stop. old = 347619328, act = 353046528 + waited 8 secs for syncstatus to stop. old = 353046528, act = 384987136 + waited 9 secs for syncstatus to stop. old = 384987136, act = 402538496 + waited 10 secs for syncstatus to stop. old = 402538496, act = 421593088 + waited 11 secs for syncstatus to stop. old = 421593088, act = 459698176 + waited 12 secs for syncstatus to stop. old = 459698176, act = 478142464 + waited 13 secs for syncstatus to stop. old = 478142464, act = 517013504 + waited 14 secs for syncstatus to stop. old = 517013504, act = 535707648 + waited 15 secs for syncstatus to stop. old = 535707648, act = 574873600 + waited 16 secs for syncstatus to stop. old = 574873600, act = 600899584 + waited 17 secs for syncstatus to stop. old = 600899584, act = 622411776 + waited 18 secs for syncstatus to stop. old = 622411776, act = 639287296 + waited 19 secs for syncstatus to stop. old = 639287296, act = 661778432 + waited 20 secs for syncstatus to stop. old = 661778432, act = 661778432 + waited 21 secs for syncstatus to stop. old = 661778432, act = 705454080 + waited 22 secs for syncstatus to stop. old = 705454080, act = 719888384 + waited 23 secs for syncstatus to stop. old = 719888384, act = 734285824 + waited 24 secs for syncstatus to stop. old = 734285824, act = 734285824 + waited 25 secs for syncstatus to stop. old = 734285824, act = 803766272 + waited 26 secs for syncstatus to stop. old = 803766272, act = 870338560 + waited 27 secs for syncstatus to stop. old = 870338560, act = 876367872 + waited 28 secs for syncstatus to stop. old = 876367872, act = 876367872 + waited 29 secs for syncstatus to stop. old = 876367872, act = 876367872 + waited 30 secs for syncstatus to stop. old = 876367872, act = 876367872 + waited 31 secs for syncstatus to stop. old = 876367872, act = 893939712 + waited 32 secs for syncstatus to stop. old = 893939712, act = 949047296 + waited 33 secs for syncstatus to stop. old = 949047296, act = 1008287744 + waited 34 secs for syncstatus to stop. old = 1008287744, act = 1029242880 + waited 35 secs for syncstatus to stop. old = 1029242880, act = 1046593536 + waited 36 secs for syncstatus to stop. old = 1046593536, act = 1063059456 + waited 37 secs for syncstatus to stop. old = 1063059456, act = 1071763456 + waited 38 secs for syncstatus to stop. old = 1071763456, act = 1100570624 + waited 39 secs for syncstatus to stop. old = 1100570624, act = 1110581248 + waited 40 secs for syncstatus to stop. old = 1110581248, act = 1129644032 + waited 41 secs for syncstatus to stop. old = 1129644032, act = 1143275520 + waited 42 secs for syncstatus to stop. old = 1143275520, act = 1159004160 + waited 43 secs for syncstatus to stop. old = 1159004160, act = 1174454272 + waited 44 secs for syncstatus to stop. old = 1174454272, act = 1188073472 + waited 45 secs for syncstatus to stop. old = 1188073472, act = 1201537024 + waited 46 secs for syncstatus to stop. old = 1201537024, act = 1222234112 + waited 47 secs for syncstatus to stop. old = 1222234112, act = 1222234112 + waited 48 secs for syncstatus to stop. old = 1222234112, act = 1253629952 + waited 49 secs for syncstatus to stop. old = 1253629952, act = 1263599616 + waited 50 secs for syncstatus to stop. old = 1263599616, act = 1283784704 + waited 51 secs for syncstatus to stop. old = 1283784704, act = 1290862592 + waited 52 secs for syncstatus to stop. old = 1290862592, act = 1316950016 + waited 53 secs for syncstatus to stop. old = 1316950016, act = 1316950016 + waited 54 secs for syncstatus to stop. old = 1316950016, act = 1349844992 + waited 55 secs for syncstatus to stop. old = 1349844992, act = 1359282176 + waited 56 secs for syncstatus to stop. old = 1359282176, act = 1372389376 + waited 57 secs for syncstatus to stop. old = 1372389376, act = 1376886784 + waited 58 secs for syncstatus to stop. old = 1376886784, act = 1408827392 + waited 59 secs for syncstatus to stop. old = 1408827392, act = 1413857280 + waited 60 secs for syncstatus to stop. old = 1413857280, act = 1413857280 + waited 61 secs for syncstatus to stop. old = 1413857280, act = 1413857280 + waited 62 secs for syncstatus to stop. old = 1413857280, act = 1413857280 + waited 63 secs for syncstatus to stop. old = 1413857280, act = 1413857280 + waited 64 secs for syncstatus to stop. old = 1413857280, act = 1413857280 + waited 65 secs for syncstatus to stop. old = 1413857280, act = 1417912320 + waited 66 secs for syncstatus to stop. old = 1417912320, act = 1438793728 + waited 67 secs for syncstatus to stop. old = 1438793728, act = 1438793728 + waited 68 secs for syncstatus to stop. old = 1438793728, act = 1501138944 + waited 69 secs for syncstatus to stop. old = 1501138944, act = 1501138944 + waited 70 secs for syncstatus to stop. old = 1501138944, act = 1571094528 + waited 71 secs for syncstatus to stop. old = 1571094528, act = 1571094528 + waited 72 secs for syncstatus to stop. old = 1571094528, act = 1614610432 + waited 73 secs for syncstatus to stop. old = 1614610432, act = 1636986880 + waited 74 secs for syncstatus to stop. old = 1636986880, act = 1657077760 + waited 75 secs for syncstatus to stop. old = 1657077760, act = 1686708224 + waited 76 secs for syncstatus to stop. old = 1686708224, act = 1700585472 + waited 77 secs for syncstatus to stop. old = 1700585472, act = 1730215936 + waited 78 secs for syncstatus to stop. old = 1730215936, act = 1737285632 + waited 79 secs for syncstatus to stop. old = 1737285632, act = 1769275392 + waited 80 secs for syncstatus to stop. old = 1769275392, act = 1807921152 + waited 81 secs for syncstatus to stop. old = 1807921152, act = 1842376704 + waited 82 secs for syncstatus to stop. old = 1842376704, act = 1860235264 + waited 83 secs for syncstatus to stop. old = 1860235264, act = 1883795456 + waited 84 secs for syncstatus to stop. old = 1883795456, act = 1896677376 + waited 85 secs for syncstatus to stop. old = 1896677376, act = 1931018240 + waited 86 secs for syncstatus to stop. old = 1931018240, act = 1931018240 + waited 87 secs for syncstatus to stop. old = 1931018240, act = 1976893440 + waited 88 secs for syncstatus to stop. old = 1976893440, act = 1976893440 + waited 89 secs for syncstatus to stop. old = 1976893440, act = 2021994496 + waited 90 secs for syncstatus to stop. old = 2021994496, act = 2041380864 + waited 91 secs for syncstatus to stop. old = 2041380864, act = 2059837440 + waited 92 secs for syncstatus to stop. old = 2059837440, act = 2059837440 + waited 93 secs for syncstatus to stop. old = 2059837440, act = 2098528256 + waited 94 secs for syncstatus to stop. old = 2098528256, act = 2098528256 + waited 95 secs for syncstatus to stop. old = 2098528256, act = 2098528256 + waited 96 secs for syncstatus to stop. old = 2098528256, act = 2098528256 + waited 97 secs for syncstatus to stop. old = 2098528256, act = 2098528256 + waited 98 secs for syncstatus to stop. old = 2098528256, act = 2098528256 + waited 99 secs for syncstatus to stop. old = 2098528256, act = 2105978880 + waited 100 secs for syncstatus to stop. old = 2105978880, act = 2147483648 + waited 101 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 102 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 103 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 104 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 105 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 106 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 107 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 108 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 109 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 110 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 111 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 112 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 113 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 114 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 115 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 116 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 117 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 118 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 119 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=11771) on istore-test-bs7 + write_count: 43 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:43, time_sync=393, rate = 6 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:15944335360, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:15983415296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:15944335360 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16019824640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:15983415296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16055635968, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16019824640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16094720000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16055635968 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16129204224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16094720000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16168730624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16129204224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16204488704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16168730624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16241602560, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16204488704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16278663168, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16241602560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16317071360, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16278663168 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16351772672, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16317071360 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16387502080, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16351772672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16423247872, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16387502080 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16456286208, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16423247872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16490643456, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16456286208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16524918784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16490643456 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16563023872, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16524918784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16597721088, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16563023872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16635277312, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16597721088 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16673660928, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16635277312 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16712478720, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16673660928 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16749281280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16712478720 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16785698816, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16749281280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16823037952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16785698816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16923553792, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16823037952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16961077248, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16923553792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:16999575552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16961077248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17038831616, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:16999575552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17076736000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17038831616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17115983872, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17076736000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17153581056, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17115983872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17192402944, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17153581056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17230917632, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17192402944 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17267273728, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17230917632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17301745664, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17267273728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17335513088, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17301745664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17369014272, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17335513088 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17405452288, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17369014272 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17440772096, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17405452288 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17475104768, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17440772096 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17514676224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17475104768 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17552359424, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17514676224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17589145600, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17552359424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17624088576, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17589145600 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17662623744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17624088576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17700376576, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17662623744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17739939840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17700376576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17786916864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17739939840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17797926912, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17786916864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17804341248, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17797926912 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17813229568, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17804341248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17817030656, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17813229568 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17822633984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17817030656 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17829961728, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17822633984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17833160704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17829961728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17845624832, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17833160704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17851265024, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17845624832 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17865195520, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17851265024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17872592896, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17865195520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17876570112, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17872592896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17880825856, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17876570112 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17885814784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17880825856 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17891004416, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17885814784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17898336256, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17891004416 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17906647040, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17898336256 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17911111680, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17906647040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17949687808, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17911111680 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:17988939776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17949687808 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18022486016, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:17988939776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18061557760, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18022486016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18101329920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18061557760 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18140098560, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18101329920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18179612672, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18140098560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18207313920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18179612672 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18246303744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18207313920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18283175936, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18246303744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18318196736, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18283175936 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18356027392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18318196736 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18391977984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18356027392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18432233472, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18391977984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18469629952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18432233472 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18507337728, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18469629952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18543452160, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18507337728 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18578161664, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18543452160 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18617192448, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18578161664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18656460800, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18617192448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 88 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18692235264, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18656460800 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 89 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18732425216, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18692235264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 90 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18769641472, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18732425216 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 91 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18806173696, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18769641472 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 92 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18843762688, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18806173696 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 93 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18880458752, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18843762688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 94 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18880458752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 95 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 96 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 97 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 98 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 99 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 100 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 101 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 102 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 103 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:18923069000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000002-istore-test-bs7,18923069000,0 + waited 2 secs for replay to stop. old = log-000000002-istore-test-bs7,18923069000,0, act = log-000000002-istore-test-bs7,18923069000,0 + waited 3 secs for replay to stop. old = log-000000002-istore-test-bs7,18923069000,0, act = log-000000002-istore-test-bs7,18923069000,0 + waited 4 secs for replay to stop. old = log-000000002-istore-test-bs7,18923069000,0, act = log-000000002-istore-test-bs7,18923069000,0 + waited 5 secs for replay to stop. old = log-000000002-istore-test-bs7,18923069000,0, act = log-000000002-istore-test-bs7,18923069000,0 + synctime = 99 + patchlength 50000, no of patches 10 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 0.76253 s, 67.1 MB/s + patching /dev/vg-mars/lv-2 at 209715 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 1.48987 s, 34.4 MB/s + patching /dev/vg-mars/lv-2 at 419430 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.17643 s, 23.5 MB/s + patching /dev/vg-mars/lv-2 at 629145 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.80164 s, 18.3 MB/s + patching /dev/vg-mars/lv-2 at 838860 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.48951 s, 14.7 MB/s + patching /dev/vg-mars/lv-2 at 1048575 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.85562 s, 13.3 MB/s + patching /dev/vg-mars/lv-2 at 1258290 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 4.86097 s, 10.5 MB/s + patching /dev/vg-mars/lv-2 at 1468005 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.04116 s, 10.2 MB/s + patching /dev/vg-mars/lv-2 at 1677720 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 6.07486 s, 8.4 MB/s + patching /dev/vg-mars/lv-2 at 1887435 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 6.28349 s, 8.1 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 15483 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 15483 1 0 14:30 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 15483 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 33042432 + waited 2 secs for syncstatus to stop. old = 33042432, act = 35897344 + waited 3 secs for syncstatus to stop. old = 35897344, act = 71581696 + waited 4 secs for syncstatus to stop. old = 71581696, act = 105598976 + waited 5 secs for syncstatus to stop. old = 105598976, act = 138412032 + waited 6 secs for syncstatus to stop. old = 138412032, act = 150028288 + waited 7 secs for syncstatus to stop. old = 150028288, act = 168181760 + waited 8 secs for syncstatus to stop. old = 168181760, act = 197459968 + waited 9 secs for syncstatus to stop. old = 197459968, act = 216580096 + waited 10 secs for syncstatus to stop. old = 216580096, act = 236208128 + waited 11 secs for syncstatus to stop. old = 236208128, act = 253034496 + waited 12 secs for syncstatus to stop. old = 253034496, act = 271237120 + waited 13 secs for syncstatus to stop. old = 271237120, act = 289210368 + waited 14 secs for syncstatus to stop. old = 289210368, act = 307822592 + waited 15 secs for syncstatus to stop. old = 307822592, act = 328060928 + waited 16 secs for syncstatus to stop. old = 328060928, act = 339398656 + waited 17 secs for syncstatus to stop. old = 339398656, act = 366501888 + waited 18 secs for syncstatus to stop. old = 366501888, act = 377479168 + waited 19 secs for syncstatus to stop. old = 377479168, act = 403505152 + waited 20 secs for syncstatus to stop. old = 403505152, act = 420773888 + waited 21 secs for syncstatus to stop. old = 420773888, act = 442302464 + waited 22 secs for syncstatus to stop. old = 442302464, act = 453963776 + waited 23 secs for syncstatus to stop. old = 453963776, act = 478826496 + waited 24 secs for syncstatus to stop. old = 478826496, act = 499449856 + waited 25 secs for syncstatus to stop. old = 499449856, act = 518078464 + waited 26 secs for syncstatus to stop. old = 518078464, act = 529596416 + waited 27 secs for syncstatus to stop. old = 529596416, act = 555409408 + waited 28 secs for syncstatus to stop. old = 555409408, act = 555409408 + waited 29 secs for syncstatus to stop. old = 555409408, act = 555409408 + waited 30 secs for syncstatus to stop. old = 555409408, act = 555409408 + waited 31 secs for syncstatus to stop. old = 555409408, act = 555409408 + waited 32 secs for syncstatus to stop. old = 555409408, act = 555409408 + waited 33 secs for syncstatus to stop. old = 555409408, act = 597508096 + waited 34 secs for syncstatus to stop. old = 597508096, act = 608645120 + waited 35 secs for syncstatus to stop. old = 608645120, act = 639713280 + waited 36 secs for syncstatus to stop. old = 639713280, act = 662765568 + waited 37 secs for syncstatus to stop. old = 662765568, act = 697782272 + waited 38 secs for syncstatus to stop. old = 697782272, act = 721244160 + waited 39 secs for syncstatus to stop. old = 721244160, act = 741785600 + waited 40 secs for syncstatus to stop. old = 741785600, act = 764440576 + waited 41 secs for syncstatus to stop. old = 764440576, act = 791515136 + waited 42 secs for syncstatus to stop. old = 791515136, act = 809517056 + waited 43 secs for syncstatus to stop. old = 809517056, act = 829976576 + waited 44 secs for syncstatus to stop. old = 829976576, act = 861179904 + waited 45 secs for syncstatus to stop. old = 861179904, act = 870068224 + waited 46 secs for syncstatus to stop. old = 870068224, act = 915410944 + waited 47 secs for syncstatus to stop. old = 915410944, act = 928182272 + waited 48 secs for syncstatus to stop. old = 928182272, act = 960057344 + waited 49 secs for syncstatus to stop. old = 960057344, act = 973213696 + waited 50 secs for syncstatus to stop. old = 973213696, act = 1012600832 + waited 51 secs for syncstatus to stop. old = 1012600832, act = 1032441856 + waited 52 secs for syncstatus to stop. old = 1032441856, act = 1065177088 + waited 53 secs for syncstatus to stop. old = 1065177088, act = 1092440064 + waited 54 secs for syncstatus to stop. old = 1092440064, act = 1125269504 + waited 55 secs for syncstatus to stop. old = 1125269504, act = 1144295424 + waited 56 secs for syncstatus to stop. old = 1144295424, act = 1166491648 + waited 57 secs for syncstatus to stop. old = 1166491648, act = 1183907840 + waited 58 secs for syncstatus to stop. old = 1183907840, act = 1201336320 + waited 59 secs for syncstatus to stop. old = 1201336320, act = 1216151552 + waited 60 secs for syncstatus to stop. old = 1216151552, act = 1226522624 + waited 61 secs for syncstatus to stop. old = 1226522624, act = 1226522624 + waited 62 secs for syncstatus to stop. old = 1226522624, act = 1226522624 + waited 63 secs for syncstatus to stop. old = 1226522624, act = 1226522624 + waited 64 secs for syncstatus to stop. old = 1226522624, act = 1233489920 + waited 65 secs for syncstatus to stop. old = 1233489920, act = 1253371904 + waited 66 secs for syncstatus to stop. old = 1253371904, act = 1261780992 + waited 67 secs for syncstatus to stop. old = 1261780992, act = 1285832704 + waited 68 secs for syncstatus to stop. old = 1285832704, act = 1285832704 + waited 69 secs for syncstatus to stop. old = 1285832704, act = 1312690176 + waited 70 secs for syncstatus to stop. old = 1312690176, act = 1325801472 + waited 71 secs for syncstatus to stop. old = 1325801472, act = 1343549440 + waited 72 secs for syncstatus to stop. old = 1343549440, act = 1343549440 + waited 73 secs for syncstatus to stop. old = 1343549440, act = 1371340800 + waited 74 secs for syncstatus to stop. old = 1371340800, act = 1386545152 + waited 75 secs for syncstatus to stop. old = 1386545152, act = 1400750080 + waited 76 secs for syncstatus to stop. old = 1400750080, act = 1419575296 + waited 77 secs for syncstatus to stop. old = 1419575296, act = 1428533248 + waited 78 secs for syncstatus to stop. old = 1428533248, act = 1428533248 + waited 79 secs for syncstatus to stop. old = 1428533248, act = 1462566912 + waited 80 secs for syncstatus to stop. old = 1462566912, act = 1478987776 + waited 81 secs for syncstatus to stop. old = 1478987776, act = 1493381120 + waited 82 secs for syncstatus to stop. old = 1493381120, act = 1511763968 + waited 83 secs for syncstatus to stop. old = 1511763968, act = 1525743616 + waited 84 secs for syncstatus to stop. old = 1525743616, act = 1547022336 + waited 85 secs for syncstatus to stop. old = 1547022336, act = 1556152320 + waited 86 secs for syncstatus to stop. old = 1556152320, act = 1578196992 + waited 87 secs for syncstatus to stop. old = 1578196992, act = 1587712000 + waited 88 secs for syncstatus to stop. old = 1587712000, act = 1606238208 + waited 89 secs for syncstatus to stop. old = 1606238208, act = 1620672512 + waited 90 secs for syncstatus to stop. old = 1620672512, act = 1641873408 + waited 91 secs for syncstatus to stop. old = 1641873408, act = 1658388480 + waited 92 secs for syncstatus to stop. old = 1658388480, act = 1664942080 + waited 93 secs for syncstatus to stop. old = 1664942080, act = 1664942080 + waited 94 secs for syncstatus to stop. old = 1664942080, act = 1664942080 + waited 95 secs for syncstatus to stop. old = 1664942080, act = 1664942080 + waited 96 secs for syncstatus to stop. old = 1664942080, act = 1664942080 + waited 97 secs for syncstatus to stop. old = 1664942080, act = 1664942080 + waited 98 secs for syncstatus to stop. old = 1664942080, act = 1715056640 + waited 99 secs for syncstatus to stop. old = 1715056640, act = 1722089472 + waited 100 secs for syncstatus to stop. old = 1722089472, act = 1756647424 + waited 101 secs for syncstatus to stop. old = 1756647424, act = 1826742272 + waited 102 secs for syncstatus to stop. old = 1826742272, act = 1882542080 + waited 103 secs for syncstatus to stop. old = 1882542080, act = 1932021760 + waited 104 secs for syncstatus to stop. old = 1932021760, act = 1989148672 + waited 105 secs for syncstatus to stop. old = 1989148672, act = 2007986176 + waited 106 secs for syncstatus to stop. old = 2007986176, act = 2044358656 + waited 107 secs for syncstatus to stop. old = 2044358656, act = 2063695872 + waited 108 secs for syncstatus to stop. old = 2063695872, act = 2092482560 + waited 109 secs for syncstatus to stop. old = 2092482560, act = 2147483648 + waited 110 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 111 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 112 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 113 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 114 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 115 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 116 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 117 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 118 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 119 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 120 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 121 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 122 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 123 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 124 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 125 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 126 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 127 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 128 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=15483) on istore-test-bs7 + write_count: 41 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:41, time_sync=453, rate = 5 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22054961152, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22093066240, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22054961152 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22131023872, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22093066240 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22166810624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22131023872 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22201110528, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22166810624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22237978624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22201110528 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22275387392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22237978624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22314770432, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22275387392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22359441408, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22314770432 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22386831360, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22359441408 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22420475904, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22386831360 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22458040320, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22420475904 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22496681984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22458040320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22532657152, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22496681984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22570913792, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22532657152 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22606233600, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22570913792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22646472704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22606233600 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22683619328, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22646472704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22718971904, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22683619328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22753587200, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22718971904 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22788427776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22753587200 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22825570304, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22788427776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22864289792, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22825570304 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22898683904, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22864289792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22934896640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22898683904 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:22973915136, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22934896640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23009710080, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:22973915136 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23049072640, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23009710080 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23084969984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23049072640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23123865600, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23084969984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23160438784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23123865600 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23196356608, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23160438784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23233421312, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23196356608 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23271067648, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23233421312 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23309545472, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23271067648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23345786880, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23309545472 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23385812992, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23345786880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23425355776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23385812992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23462703104, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23425355776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23500767232, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23462703104 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23538634752, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23500767232 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23575584768, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23538634752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23613202432, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23575584768 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23649701888, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23613202432 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23687000064, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23649701888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23724085248, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23687000064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23759077376, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23724085248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23796908032, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23759077376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23832408064, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23796908032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23871139840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23832408064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23907753984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23871139840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23945613312, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23907753984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:23984955392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23945613312 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24021491712, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:23984955392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24054046720, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24021491712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24092291072, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24054046720 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24131813376, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24092291072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24169967616, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24131813376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24209084416, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24169967616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24247566336, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24209084416 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24287342592, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24247566336 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24325570560, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24287342592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24361639936, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24325570560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24399671296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24361639936 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24439361536, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24399671296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24478490624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24439361536 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24517918720, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24478490624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24552697856, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24517918720 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24590589952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24552697856 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24629698560, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24590589952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24670441472, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24629698560 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24704987136, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24670441472 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24741191680, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24704987136 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24778240000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24741191680 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24816619520, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24778240000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24853704704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24816619520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898562920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24853704704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898562920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:24898571280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000002-istore-test-bs7,24898571280,0 + waited 2 secs for replay to stop. old = log-000000002-istore-test-bs7,24898571280,0, act = log-000000002-istore-test-bs7,24898571280,0 + waited 3 secs for replay to stop. old = log-000000002-istore-test-bs7,24898571280,0, act = log-000000002-istore-test-bs7,24898571280,0 + waited 4 secs for replay to stop. old = log-000000002-istore-test-bs7,24898571280,0, act = log-000000002-istore-test-bs7,24898571280,0 + waited 5 secs for replay to stop. old = log-000000002-istore-test-bs7,24898571280,0, act = log-000000002-istore-test-bs7,24898571280,0 + synctime = 108 + patchlength 500000, no of patches 4 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 7.28786 s, 70.3 MB/s + patching /dev/vg-mars/lv-2 at 524288 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 8.6359 s, 59.3 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 10.2347 s, 50.0 MB/s + patching /dev/vg-mars/lv-2 at 1572864 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 11.6448 s, 44.0 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 18869 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 18869 1 0 14:41 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 18869 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 35704832 + waited 2 secs for syncstatus to stop. old = 35704832, act = 64815104 + waited 3 secs for syncstatus to stop. old = 64815104, act = 68354048 + waited 4 secs for syncstatus to stop. old = 68354048, act = 131149824 + waited 5 secs for syncstatus to stop. old = 131149824, act = 149659648 + waited 6 secs for syncstatus to stop. old = 149659648, act = 179023872 + waited 7 secs for syncstatus to stop. old = 179023872, act = 209485824 + waited 8 secs for syncstatus to stop. old = 209485824, act = 226971648 + waited 9 secs for syncstatus to stop. old = 226971648, act = 261816320 + waited 10 secs for syncstatus to stop. old = 261816320, act = 288694272 + waited 11 secs for syncstatus to stop. old = 288694272, act = 399335424 + waited 12 secs for syncstatus to stop. old = 399335424, act = 412430336 + waited 13 secs for syncstatus to stop. old = 412430336, act = 414318592 + waited 14 secs for syncstatus to stop. old = 414318592, act = 415039488 + waited 15 secs for syncstatus to stop. old = 415039488, act = 419024896 + waited 16 secs for syncstatus to stop. old = 419024896, act = 419356672 + waited 17 secs for syncstatus to stop. old = 419356672, act = 422014976 + waited 18 secs for syncstatus to stop. old = 422014976, act = 423600128 + waited 19 secs for syncstatus to stop. old = 423600128, act = 425684992 + waited 20 secs for syncstatus to stop. old = 425684992, act = 427753472 + waited 21 secs for syncstatus to stop. old = 427753472, act = 429703168 + waited 22 secs for syncstatus to stop. old = 429703168, act = 449904640 + waited 23 secs for syncstatus to stop. old = 449904640, act = 463069184 + waited 24 secs for syncstatus to stop. old = 463069184, act = 530702336 + waited 25 secs for syncstatus to stop. old = 530702336, act = 544579584 + waited 26 secs for syncstatus to stop. old = 544579584, act = 567697408 + waited 27 secs for syncstatus to stop. old = 567697408, act = 567697408 + waited 28 secs for syncstatus to stop. old = 567697408, act = 567697408 + waited 29 secs for syncstatus to stop. old = 567697408, act = 567697408 + waited 30 secs for syncstatus to stop. old = 567697408, act = 567697408 + waited 31 secs for syncstatus to stop. old = 567697408, act = 587481088 + waited 32 secs for syncstatus to stop. old = 587481088, act = 618283008 + waited 33 secs for syncstatus to stop. old = 618283008, act = 630259712 + waited 34 secs for syncstatus to stop. old = 630259712, act = 641753088 + waited 35 secs for syncstatus to stop. old = 641753088, act = 649695232 + waited 36 secs for syncstatus to stop. old = 649695232, act = 667484160 + waited 37 secs for syncstatus to stop. old = 667484160, act = 684261376 + waited 38 secs for syncstatus to stop. old = 684261376, act = 696582144 + waited 39 secs for syncstatus to stop. old = 696582144, act = 699998208 + waited 40 secs for syncstatus to stop. old = 699998208, act = 723410944 + waited 41 secs for syncstatus to stop. old = 723410944, act = 737476608 + waited 42 secs for syncstatus to stop. old = 737476608, act = 752418816 + waited 43 secs for syncstatus to stop. old = 752418816, act = 766537728 + waited 44 secs for syncstatus to stop. old = 766537728, act = 779829248 + waited 45 secs for syncstatus to stop. old = 779829248, act = 788656128 + waited 46 secs for syncstatus to stop. old = 788656128, act = 809574400 + waited 47 secs for syncstatus to stop. old = 809574400, act = 823984128 + waited 48 secs for syncstatus to stop. old = 823984128, act = 836497408 + waited 49 secs for syncstatus to stop. old = 836497408, act = 849956864 + waited 50 secs for syncstatus to stop. old = 849956864, act = 863670272 + waited 51 secs for syncstatus to stop. old = 863670272, act = 872218624 + waited 52 secs for syncstatus to stop. old = 872218624, act = 891879424 + waited 53 secs for syncstatus to stop. old = 891879424, act = 896184320 + waited 54 secs for syncstatus to stop. old = 896184320, act = 922025984 + waited 55 secs for syncstatus to stop. old = 922025984, act = 935133184 + waited 56 secs for syncstatus to stop. old = 935133184, act = 949551104 + waited 57 secs for syncstatus to stop. old = 949551104, act = 958201856 + waited 58 secs for syncstatus to stop. old = 958201856, act = 978649088 + waited 59 secs for syncstatus to stop. old = 978649088, act = 987447296 + waited 60 secs for syncstatus to stop. old = 987447296, act = 987447296 + waited 61 secs for syncstatus to stop. old = 987447296, act = 987447296 + waited 62 secs for syncstatus to stop. old = 987447296, act = 987447296 + waited 63 secs for syncstatus to stop. old = 987447296, act = 987447296 + waited 64 secs for syncstatus to stop. old = 987447296, act = 987447296 + waited 65 secs for syncstatus to stop. old = 987447296, act = 997523456 + waited 66 secs for syncstatus to stop. old = 997523456, act = 1025372160 + waited 67 secs for syncstatus to stop. old = 1025372160, act = 1062625280 + waited 68 secs for syncstatus to stop. old = 1062625280, act = 1114771456 + waited 69 secs for syncstatus to stop. old = 1114771456, act = 1160691712 + waited 70 secs for syncstatus to stop. old = 1160691712, act = 1211170816 + waited 71 secs for syncstatus to stop. old = 1211170816, act = 1240473600 + waited 72 secs for syncstatus to stop. old = 1240473600, act = 1249644544 + waited 73 secs for syncstatus to stop. old = 1249644544, act = 1308811264 + waited 74 secs for syncstatus to stop. old = 1308811264, act = 1341751296 + waited 75 secs for syncstatus to stop. old = 1341751296, act = 1399291904 + waited 76 secs for syncstatus to stop. old = 1399291904, act = 1399291904 + waited 77 secs for syncstatus to stop. old = 1399291904, act = 1505443840 + waited 78 secs for syncstatus to stop. old = 1505443840, act = 1573203968 + waited 79 secs for syncstatus to stop. old = 1573203968, act = 1655734272 + waited 80 secs for syncstatus to stop. old = 1655734272, act = 1667915776 + waited 81 secs for syncstatus to stop. old = 1667915776, act = 1672454144 + waited 82 secs for syncstatus to stop. old = 1672454144, act = 1674227712 + waited 83 secs for syncstatus to stop. old = 1674227712, act = 1676054528 + waited 84 secs for syncstatus to stop. old = 1676054528, act = 1677606912 + waited 85 secs for syncstatus to stop. old = 1677606912, act = 1679687680 + waited 86 secs for syncstatus to stop. old = 1679687680, act = 1685913600 + waited 87 secs for syncstatus to stop. old = 1685913600, act = 1735053312 + waited 88 secs for syncstatus to stop. old = 1735053312, act = 1821372416 + waited 89 secs for syncstatus to stop. old = 1821372416, act = 1849884672 + waited 90 secs for syncstatus to stop. old = 1849884672, act = 1955237888 + waited 91 secs for syncstatus to stop. old = 1955237888, act = 1986015232 + waited 92 secs for syncstatus to stop. old = 1986015232, act = 2054459392 + waited 93 secs for syncstatus to stop. old = 2054459392, act = 2070872064 + waited 94 secs for syncstatus to stop. old = 2070872064, act = 2070872064 + waited 95 secs for syncstatus to stop. old = 2070872064, act = 2070872064 + waited 96 secs for syncstatus to stop. old = 2070872064, act = 2070872064 + waited 97 secs for syncstatus to stop. old = 2070872064, act = 2070872064 + waited 98 secs for syncstatus to stop. old = 2070872064, act = 2083373056 + waited 99 secs for syncstatus to stop. old = 2083373056, act = 2083373056 + waited 100 secs for syncstatus to stop. old = 2083373056, act = 2142121984 + waited 101 secs for syncstatus to stop. old = 2142121984, act = 2147483648 + waited 102 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 103 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 104 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 105 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 106 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 107 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 108 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 109 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 110 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 111 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 112 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 113 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 114 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 115 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 116 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 117 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 118 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 119 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 120 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=18869) on istore-test-bs7 + write_count: 44 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:44, time_sync=397, rate = 6 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27638661120, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27674337280, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27638661120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27711401984, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27674337280 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27746349056, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27711401984 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27787595776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27746349056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27825352704, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27787595776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27862949888, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27825352704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27902561720, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27862949888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27932450816, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27902561720 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:27974098944, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27932450816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28013277184, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:27974098944 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28050563072, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28013277184 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28089380864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28050563072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28128002048, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28089380864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28164481024, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28128002048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28203671552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28164481024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28241620992, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28203671552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28279480320, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28241620992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28319211520, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28279480320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28358012928, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28319211520 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28392738816, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28358012928 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28427984896, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28392738816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28464492544, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28427984896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28501069824, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28464492544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28538019840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28501069824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28577333248, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28538019840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28617752576, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28577333248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28657987584, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28617752576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28697595904, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28657987584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28735021056, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28697595904 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28770680832, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28735021056 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28807749632, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28770680832 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28845543424, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28807749632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28884086784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28845543424 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28923424768, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28884086784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:28961624064, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28923424768 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29000192000, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:28961624064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29039214592, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29000192000 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29071790080, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29039214592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29107613696, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29071790080 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29148131328, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29107613696 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29187485696, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29148131328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29222141952, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29187485696 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29258530816, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29222141952 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29298999296, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29258530816 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29333602304, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29298999296 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29373517824, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29333602304 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29411966976, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29373517824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29448675328, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29411966976 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29483368448, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29448675328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29523628032, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29483368448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29562327040, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29523628032 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29600530432, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29562327040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29638836224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29600530432 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29674139648, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29638836224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29711720448, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29674139648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29747638272, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29711720448 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29784940544, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29747638272 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29821034496, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29784940544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29860929536, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29821034496 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29892427776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29860929536 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29929549824, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29892427776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:29967446016, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29929549824 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30005329920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:29967446016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30044192768, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30005329920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30083702784, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30044192768 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30122143744, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30083702784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30161723392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30122143744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30200901632, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30161723392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30233268224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30200901632 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30271844352, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30233268224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30312058880, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30271844352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30347395072, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30312058880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30387236864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30347395072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30425681920, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30387236864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30464114688, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30425681920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30502817792, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30464114688 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30540746752, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30502817792 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30577217536, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30540746752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30617640960, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30577217536 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30655475712, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30617640960 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 82 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30693838848, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30655475712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 83 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30727622656, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30693838848 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 84 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30765789184, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30727622656 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 85 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30801211392, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30765789184 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 86 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30837207040, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30801211392 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 87 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30874021888, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30837207040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 88 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30911651840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30874021888 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 89 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30949978112, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30911651840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 90 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:30990618624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30949978112 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 91 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31031156736, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:30990618624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 92 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31072915456, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31031156736 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 93 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31109734400, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31072915456 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 94 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31143706624, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31109734400 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 95 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31182004224, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31143706624 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 96 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31218237440, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31182004224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 97 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31252209664, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31218237440 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 98 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31292133376, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31252209664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 99 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31329374208, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31292133376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 100 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31367491584, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31329374208 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 101 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31402725376, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31367491584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 102 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31439618048, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31402725376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 103 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31479816192, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31439618048 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 104 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31513059328, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31479816192 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 105 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31548526592, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31513059328 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 106 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31585468416, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31548526592 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 107 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31623020544, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31585468416 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 108 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31657390080, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31623020544 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 109 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31694864384, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31657390080 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 110 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31728517120, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31694864384 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 111 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31761862656, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31728517120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 112 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31797911552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31761862656 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 113 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31833124864, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31797911552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 114 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31869739008, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31833124864 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 115 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31906967552, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31869739008 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 116 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31929933200, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31906967552 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 117 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31929933200 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 118 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 119 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 120 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 121 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 122 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 123 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 124 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 125 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 126 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:31939538840 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000002-istore-test-bs7,31939538840,0 + waited 2 secs for replay to stop. old = log-000000002-istore-test-bs7,31939538840,0, act = log-000000002-istore-test-bs7,31939538840,0 + waited 3 secs for replay to stop. old = log-000000002-istore-test-bs7,31939538840,0, act = log-000000002-istore-test-bs7,31939538840,0 + waited 4 secs for replay to stop. old = log-000000002-istore-test-bs7,31939538840,0, act = log-000000002-istore-test-bs7,31939538840,0 + waited 5 secs for replay to stop. old = log-000000002-istore-test-bs7,31939538840,0, act = log-000000002-istore-test-bs7,31939538840,0 + synctime = 100 + patchlength 1000000, no of patches 2 + executing marsadm --timeout=30 down lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started pause-replay-local on resource lv-2 +successfully started pause-sync-local on resource lv-2 +successfully started detach on resource lv-2 +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '0'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '0' +device '/dev/mars/lv-2' is no longer present + patching /dev/vg-mars/lv-2 at 0 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 14.6478 s, 69.9 MB/s + patching /dev/vg-mars/lv-2 at 1048576 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 17.3017 s, 59.2 MB/s + executing marsadm --timeout=30 invalidate lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- stop old replay: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 +---------------------------- wait for replay off: +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-syncing' has acceptable value '0' +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-replaying' has acceptable value '0' +---------------------------- force symlinks: +successfully started invalidate on resource lv-2 +successfully started invalidate on resource lv-2 + checking directories /mnt/test and /lib on istore-test-bs7 + copying script /tmp/write_and_delete.sh.13181 to istore-test-bs7:/tmp/write_and_delete.sh.13181 + executing /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err on istore-test-bs7 + checking whether process 22602 is running on istore-test-bs7 +UID PID PPID C STIME TTY TIME CMD +root 22602 1 0 14:53 ? 00:00:00 bash -c /tmp/write_and_delete.sh.13181 >/tmp/write_and_delete.sh.13181.out 2>/tmp/write_and_delete.sh.13181.err & echo $! + checking whether process 22602 has errors in /tmp/write_and_delete.sh.13181.err + executing marsadm --timeout=30 up lv-2 on istore-test-bap7 +---------------------------- check preconditions: +---------------------------- switch state: +successfully started attach on resource lv-2 +connect on resource lv-2 is already activated +resume-sync-local on resource lv-2 is already activated +resume-replay-local on resource lv-2 is already activated +---------------------------- wait for effect: +waiting until 'is-attached' reaches the value '1'... +OK, '/mars/resource-lv-2/actual-istore-test-bap7/is-attached' has acceptable value '1' + waited 1 secs for syncstatus to stop. old = x, act = 75870208 + waited 2 secs for syncstatus to stop. old = 75870208, act = 217219072 + waited 3 secs for syncstatus to stop. old = 217219072, act = 231297024 + waited 4 secs for syncstatus to stop. old = 231297024, act = 260206592 + waited 5 secs for syncstatus to stop. old = 260206592, act = 278462464 + waited 6 secs for syncstatus to stop. old = 278462464, act = 298024960 + waited 7 secs for syncstatus to stop. old = 298024960, act = 305950720 + waited 8 secs for syncstatus to stop. old = 305950720, act = 343138304 + waited 9 secs for syncstatus to stop. old = 343138304, act = 381104128 + waited 10 secs for syncstatus to stop. old = 381104128, act = 443777024 + waited 11 secs for syncstatus to stop. old = 443777024, act = 480292864 + waited 12 secs for syncstatus to stop. old = 480292864, act = 485138432 + waited 13 secs for syncstatus to stop. old = 485138432, act = 528596992 + waited 14 secs for syncstatus to stop. old = 528596992, act = 564748288 + waited 15 secs for syncstatus to stop. old = 564748288, act = 626622464 + waited 16 secs for syncstatus to stop. old = 626622464, act = 626622464 + waited 17 secs for syncstatus to stop. old = 626622464, act = 728236032 + waited 18 secs for syncstatus to stop. old = 728236032, act = 799227904 + waited 19 secs for syncstatus to stop. old = 799227904, act = 831356928 + waited 20 secs for syncstatus to stop. old = 831356928, act = 831356928 + waited 21 secs for syncstatus to stop. old = 831356928, act = 900239360 + waited 22 secs for syncstatus to stop. old = 900239360, act = 924647424 + waited 23 secs for syncstatus to stop. old = 924647424, act = 962027520 + waited 24 secs for syncstatus to stop. old = 962027520, act = 962027520 + waited 25 secs for syncstatus to stop. old = 962027520, act = 1082863616 + waited 26 secs for syncstatus to stop. old = 1082863616, act = 1127641088 + waited 27 secs for syncstatus to stop. old = 1127641088, act = 1191514112 + waited 28 secs for syncstatus to stop. old = 1191514112, act = 1191514112 + waited 29 secs for syncstatus to stop. old = 1191514112, act = 1191514112 + waited 30 secs for syncstatus to stop. old = 1191514112, act = 1198579712 + waited 31 secs for syncstatus to stop. old = 1198579712, act = 1221693440 + waited 32 secs for syncstatus to stop. old = 1221693440, act = 1288138752 + waited 33 secs for syncstatus to stop. old = 1288138752, act = 1318699008 + waited 34 secs for syncstatus to stop. old = 1318699008, act = 1342414848 + waited 35 secs for syncstatus to stop. old = 1342414848, act = 1377796096 + waited 36 secs for syncstatus to stop. old = 1377796096, act = 1377796096 + waited 37 secs for syncstatus to stop. old = 1377796096, act = 1459900416 + waited 38 secs for syncstatus to stop. old = 1459900416, act = 1483751424 + waited 39 secs for syncstatus to stop. old = 1483751424, act = 1511649280 + waited 40 secs for syncstatus to stop. old = 1511649280, act = 1532932096 + waited 41 secs for syncstatus to stop. old = 1532932096, act = 1549074432 + waited 42 secs for syncstatus to stop. old = 1549074432, act = 1549074432 + waited 43 secs for syncstatus to stop. old = 1549074432, act = 1589182464 + waited 44 secs for syncstatus to stop. old = 1589182464, act = 1589182464 + waited 45 secs for syncstatus to stop. old = 1589182464, act = 1589182464 + waited 46 secs for syncstatus to stop. old = 1589182464, act = 1634009088 + waited 47 secs for syncstatus to stop. old = 1634009088, act = 1634009088 + waited 48 secs for syncstatus to stop. old = 1634009088, act = 1677234176 + waited 49 secs for syncstatus to stop. old = 1677234176, act = 1682329600 + waited 50 secs for syncstatus to stop. old = 1682329600, act = 1682329600 + waited 51 secs for syncstatus to stop. old = 1682329600, act = 1727152128 + waited 52 secs for syncstatus to stop. old = 1727152128, act = 1727152128 + waited 53 secs for syncstatus to stop. old = 1727152128, act = 1763508224 + waited 54 secs for syncstatus to stop. old = 1763508224, act = 1768751104 + waited 55 secs for syncstatus to stop. old = 1768751104, act = 1768751104 + waited 56 secs for syncstatus to stop. old = 1768751104, act = 1813839872 + waited 57 secs for syncstatus to stop. old = 1813839872, act = 1813839872 + waited 58 secs for syncstatus to stop. old = 1813839872, act = 1813839872 + waited 59 secs for syncstatus to stop. old = 1813839872, act = 1813839872 + waited 60 secs for syncstatus to stop. old = 1813839872, act = 1813839872 + waited 61 secs for syncstatus to stop. old = 1813839872, act = 1813839872 + waited 62 secs for syncstatus to stop. old = 1813839872, act = 1813839872 + waited 63 secs for syncstatus to stop. old = 1813839872, act = 1825894400 + waited 64 secs for syncstatus to stop. old = 1825894400, act = 1857593344 + waited 65 secs for syncstatus to stop. old = 1857593344, act = 1910620160 + waited 66 secs for syncstatus to stop. old = 1910620160, act = 1930145792 + waited 67 secs for syncstatus to stop. old = 1930145792, act = 1955835904 + waited 68 secs for syncstatus to stop. old = 1955835904, act = 1997864960 + waited 69 secs for syncstatus to stop. old = 1997864960, act = 2017153024 + waited 70 secs for syncstatus to stop. old = 2017153024, act = 2038710272 + waited 71 secs for syncstatus to stop. old = 2038710272, act = 2050875392 + waited 72 secs for syncstatus to stop. old = 2050875392, act = 2085941248 + waited 73 secs for syncstatus to stop. old = 2085941248, act = 2091900928 + waited 74 secs for syncstatus to stop. old = 2091900928, act = 2126266368 + waited 75 secs for syncstatus to stop. old = 2126266368, act = 2132299776 + waited 76 secs for syncstatus to stop. old = 2132299776, act = 2147483648 + waited 77 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 78 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 79 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 80 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 81 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 82 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 83 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 84 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 85 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 86 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 87 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 88 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 89 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 90 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 91 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 92 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 93 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 94 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + waited 95 secs for syncstatus to stop. old = 2147483648, act = 2147483648 + determine pid of script /tmp/write_and_delete.sh.13181 on istore-test-bs7 + stopping script /tmp/write_and_delete.sh.13181 (pid=22602) on istore-test-bs7 + write_count: 36 + removing files /tmp/write_and_delete.sh.13181* on istore-test-bs7 +write_count:36, time_sync=302, rate = 7 / min + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 1 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34220224512, old = x + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 2 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34260299776, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34220224512 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 3 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34296020992, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34260299776 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 4 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34334773248, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34296020992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 5 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34372145152, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34334773248 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 6 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34408734720, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34372145152 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000002-istore-test-bs7 + waited 7 secs for /mars/resource-lv-2/log-000000002-istore-test-bs7 act = /mars/resource-lv-2/log-000000002-istore-test-bs7:34429719500, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34408734720 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 8 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:38043648, old = /mars/resource-lv-2/log-000000002-istore-test-bs7:34429719500 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 9 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:77807616, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:38043648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 10 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:115073024, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:77807616 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 11 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:150487040, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:115073024 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 12 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:188760064, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:150487040 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 13 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:223899648, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:188760064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 14 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:261275648, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:223899648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 15 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:299470848, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:261275648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 16 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:337432576, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:299470848 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 17 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:378097664, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:337432576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 18 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:415698944, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:378097664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 19 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:451760128, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:415698944 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 20 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:489684992, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:451760128 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 21 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:525148160, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:489684992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 22 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:561213440, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:525148160 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 23 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:595808256, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:561213440 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 24 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:630079488, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:595808256 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 25 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:667697152, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:630079488 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 26 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:704634880, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:667697152 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 27 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:741679104, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:704634880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 28 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:781496320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:741679104 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 29 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:821460992, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:781496320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 30 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:859123712, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:821460992 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 31 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:899538944, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:859123712 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 32 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:938164224, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:899538944 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 33 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:972644352, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:938164224 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 34 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1006747648, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:972644352 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 35 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1045872640, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1006747648 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 36 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1086816256, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1045872640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 37 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1128177664, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1086816256 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 38 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1165656064, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1128177664 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 39 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1204822016, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1165656064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 40 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1241169920, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1204822016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 41 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1278083072, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1241169920 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 42 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1316982784, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1278083072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 43 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1356005376, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1316982784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 44 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1394376704, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1356005376 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 45 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1430917120, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1394376704 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 46 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1469489152, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1430917120 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 47 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1509535744, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1469489152 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 48 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1547608064, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1509535744 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 49 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1583792128, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1547608064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 50 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1618776064, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1583792128 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 51 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1655754752, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1618776064 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 52 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1692942336, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1655754752 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 53 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1732694016, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1692942336 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 54 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1771307008, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1732694016 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 55 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1810243584, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1771307008 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 56 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1848954880, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1810243584 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 57 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1885392896, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1848954880 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 58 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1923305472, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1885392896 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 59 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:1960181760, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1923305472 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 60 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2000408576, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:1960181760 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 61 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2038882304, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2000408576 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 62 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2078064640, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2038882304 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 63 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2116235264, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2078064640 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 64 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2153078784, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2116235264 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 65 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2190745600, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2153078784 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 66 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2228662272, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2190745600 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 67 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2266873856, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2228662272 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 68 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2304262144, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2266873856 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 69 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2342531072, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2304262144 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 70 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2377454420, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2342531072 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 71 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402131968, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2377454420 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 72 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402131968 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 73 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 74 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 75 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 76 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 77 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 78 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 79 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 80 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 81 secs for /mars/resource-lv-2/log-000000003-istore-test-bs7 act = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320, old = /mars/resource-lv-2/log-000000003-istore-test-bs7:2402346320 + get length of istore-test-bap7:/mars/resource-lv-2/log-000000003-istore-test-bs7 + waited 1 secs for replay to stop. old = x, act = log-000000003-istore-test-bs7,2402346320,0 + waited 2 secs for replay to stop. old = log-000000003-istore-test-bs7,2402346320,0, act = log-000000003-istore-test-bs7,2402346320,0 + waited 3 secs for replay to stop. old = log-000000003-istore-test-bs7,2402346320,0, act = log-000000003-istore-test-bs7,2402346320,0 + waited 4 secs for replay to stop. old = log-000000003-istore-test-bs7,2402346320,0, act = log-000000003-istore-test-bs7,2402346320,0 + waited 5 secs for replay to stop. old = log-000000003-istore-test-bs7,2402346320,0, act = log-000000003-istore-test-bs7,2402346320,0 + synctime = 75 +use_mars_sync=1 synctimes = 99 108 100 75 +==> Thu Jul 18 15:01:25 CEST 2013 Finished. +=============================================================== + + +=============================================================== +======== ./rsync +Sourcing config file /home/fl/mars/test_suite/user_modules.conf +Sourcing config file /home/fl/mars/test_suite/test_cases/perf/sync/rsync.conf +==> Thu Jul 18 15:01:25 CEST 2013 Starting .rsync +calling synctest_run + generating file istore-test-bs7:/mars/file_to_sync (2097152 KB) from /dev/vg-mars/lv-2 +2097152+0 records in +2097152+0 records out +2147483648 bytes (2.1 GB) copied, 37.8235 s, 56.8 MB/s + syncing istore-test-bs7:/mars/file_to_sync to istore-test-bap7 +receiving incremental file list +file_to_sync + +sent 30 bytes received 2147745875 bytes 64111818.06 bytes/sec +total size is 2147483648 speedup is 1.00 +-rw-r--r-- 1 root root 2147483648 Jul 18 15:02 /mars/file_to_sync +-rw-r--r-- 1 root root 2147483648 Jul 18 15:02 /mars/file_to_sync + patchlength 1, no of patches 10 + patching /mars/file_to_sync at 0 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.000149445 s, 6.9 MB/s + patching /mars/file_to_sync at 209715 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 0.762688 s, 1.3 kB/s + patching /mars/file_to_sync at 419430 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 1.51781 s, 0.7 kB/s + patching /mars/file_to_sync at 629145 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.0743 s, 0.5 kB/s + patching /mars/file_to_sync at 838860 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 2.86588 s, 0.4 kB/s + patching /mars/file_to_sync at 1048575 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.40777 s, 0.3 kB/s + patching /mars/file_to_sync at 1258290 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 3.75502 s, 0.3 kB/s + patching /mars/file_to_sync at 1468005 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 4.3656 s, 0.2 kB/s + patching /mars/file_to_sync at 1677720 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 4.97989 s, 0.2 kB/s + patching /mars/file_to_sync at 1887435 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 5.69627 s, 0.2 kB/s + patching /mars/file_to_sync at 2097150 KB with 1 KB +1+0 records in +1+0 records out +1024 bytes (1.0 kB) copied, 6.22759 s, 0.2 kB/s + syncing istore-test-bs7:/mars/file_to_sync to istore-test-bap7 +receiving incremental file list +file_to_sync + +sent 42 bytes received 2147745875 bytes 50535198.05 bytes/sec +total size is 2147483648 speedup is 1.00 + synctime = 84 + patchlength 50000, no of patches 10 + patching /mars/file_to_sync at 0 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 0.441773 s, 116 MB/s + patching /mars/file_to_sync at 209715 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 0.957382 s, 53.5 MB/s + patching /mars/file_to_sync at 419430 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 1.54841 s, 33.1 MB/s + patching /mars/file_to_sync at 629145 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.24438 s, 22.8 MB/s + patching /mars/file_to_sync at 838860 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 2.89929 s, 17.7 MB/s + patching /mars/file_to_sync at 1048575 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.56201 s, 14.4 MB/s + patching /mars/file_to_sync at 1258290 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 3.99976 s, 12.8 MB/s + patching /mars/file_to_sync at 1468005 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 4.58376 s, 11.2 MB/s + patching /mars/file_to_sync at 1677720 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.19575 s, 9.9 MB/s + patching /mars/file_to_sync at 1887435 KB with 50000 KB +50000+0 records in +50000+0 records out +51200000 bytes (51 MB) copied, 5.92371 s, 8.6 MB/s + syncing istore-test-bs7:/mars/file_to_sync to istore-test-bap7 +receiving incremental file list +file_to_sync + +sent 50143 bytes received 2147745875 bytes 47204308.09 bytes/sec +total size is 2147483648 speedup is 1.00 + synctime = 86 + patchlength 500000, no of patches 4 + patching /mars/file_to_sync at 0 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 2.96797 s, 173 MB/s + patching /mars/file_to_sync at 524288 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 3.30976 s, 155 MB/s + patching /mars/file_to_sync at 1048576 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 4.82674 s, 106 MB/s + patching /mars/file_to_sync at 1572864 KB with 500000 KB +500000+0 records in +500000+0 records out +512000000 bytes (512 MB) copied, 6.42652 s, 79.7 MB/s + syncing istore-test-bs7:/mars/file_to_sync to istore-test-bap7 +receiving incremental file list +file_to_sync + +sent 158447 bytes received 2147745875 bytes 38700978.77 bytes/sec +total size is 2147483648 speedup is 1.00 + synctime = 96 + patchlength 1000000, no of patches 2 + patching /mars/file_to_sync at 0 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 5.88114 s, 174 MB/s + patching /mars/file_to_sync at 1048576 KB with 1000000 KB +1000000+0 records in +1000000+0 records out +1024000000 bytes (1.0 GB) copied, 6.11854 s, 167 MB/s + syncing istore-test-bs7:/mars/file_to_sync to istore-test-bap7 +receiving incremental file list +file_to_sync + +sent 224030 bytes received 2147745875 bytes 41708153.50 bytes/sec +total size is 2147483648 speedup is 1.00 + synctime = 92 +use_mars_sync=0 synctimes = 84 86 96 92 +calling lib_err_check_and_move_global_err_files_all +/home/fl/mars/test_suite/scripts/modules/00_main.sh: line 26: lib_err_check_and_move_global_err_files_all: command not found +Replay failure 1 +Failure 255 diff --git a/test_suite/test_cases/hardcore/aio_filehandle/aio_filehandle.conf b/test_suite/test_cases/bugs/aio_filehandle/aio_filehandle.conf similarity index 100% rename from test_suite/test_cases/hardcore/aio_filehandle/aio_filehandle.conf rename to test_suite/test_cases/bugs/aio_filehandle/aio_filehandle.conf diff --git a/test_suite/test_cases/bugs/aio_filehandle/out/out.conf b/test_suite/test_cases/bugs/aio_filehandle/out/out.conf new file mode 100644 index 00000000..a68701b1 --- /dev/null +++ b/test_suite/test_cases/bugs/aio_filehandle/out/out.conf @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +## sleep time between marsadm log-rotate commands +remote_dev_log_rotate_sleep=5 + +## sleep time between marsadm log-delete commands +remote_dev_log_delete_sleep=20 + +## name of non existant file +remote_dev_non_existant_file='/tmp/i_am_not_there' + +## egrep pattern to identify error messages created by trying to access the +## remote file which does not exist +remote_dev_errmsg_pattern=' '"$main_mars_errmsg_prefix"' .*( file .*'"$remote_dev_non_existant_file"'.*|status = -2 *)$' + +## number of error messages to wait for +remote_dev_number_errmsg_req=100 + +## maxtime to wait for requested number of error messages +remote_dev_maxtime_to_wait_for_errmsg=600 + +## see comment in default-crash.conf +crash_print_linktree_during_reboot=0 + +run_list="resource_prepare resource_run remote_dev_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/bugs/memleak/memleak.conf b/test_suite/test_cases/bugs/memleak/memleak.conf new file mode 100644 index 00000000..e5447e2a --- /dev/null +++ b/test_suite/test_cases/bugs/memleak/memleak.conf @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +## Parameters for the memleak - Test + +verbose_script=1 + +resource_name_list=(lv-1-2) + +resource_set_globals + +resource_recreate_fs_on_data_device_required=0 + +run_list="resource_prepare memleak_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/bugs/memleak/out/out.conf b/test_suite/test_cases/bugs/memleak/out/out.conf new file mode 100644 index 00000000..e5447e2a --- /dev/null +++ b/test_suite/test_cases/bugs/memleak/out/out.conf @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +## Parameters for the memleak - Test + +verbose_script=1 + +resource_name_list=(lv-1-2) + +resource_set_globals + +resource_recreate_fs_on_data_device_required=0 + +run_list="resource_prepare memleak_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/hardcore/aio_filehandle/out/out.conf b/test_suite/test_cases/hardcore/aio_filehandle/out/out.conf deleted file mode 120000 index 8c177f58..00000000 --- a/test_suite/test_cases/hardcore/aio_filehandle/out/out.conf +++ /dev/null @@ -1 +0,0 @@ -../aio_filehandle.conf \ No newline at end of file diff --git a/test_suite/test_cases/hardcore/mars_dir_full/mars_dir_full.conf b/test_suite/test_cases/hardcore/mars_dir_full/mars_dir_full.conf index bf52c351..9a2896b1 100644 --- a/test_suite/test_cases/hardcore/mars_dir_full/mars_dir_full.conf +++ b/test_suite/test_cases/hardcore/mars_dir_full/mars_dir_full.conf @@ -23,6 +23,11 @@ verbose_script=1 resource_recreate_fs_on_data_device_required=0 +## maxtime to wait for secondary to become disk state = Uptodate and +## repl state = Replaying +mars_dir_full_maxtime_state_constant=60 + + cluster_mars_dir_lv_name_list=([${main_host_list[0]}]=lv-5-10 [${main_host_list[1]}]=lv-6-100) -run_list="cluster_run resource_run resource_fill_data_device lib_err_check_and_move_global_err_files_all" +run_list="cluster_run resource_run resource_fill_mars_dir lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/multiresource/multiresource.conf b/test_suite/test_cases/multiresource/multiresource.conf new file mode 100644 index 00000000..84f4b84f --- /dev/null +++ b/test_suite/test_cases/multiresource/multiresource.conf @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +# lvs lv-1-700 ... lv-8-700 +lv_config_name_list=($(eval echo lv-{1..$multires_lv_count}-$multires_lv_size)) + +# lvs lv-1-700 ... lv-7-700 +resource_name_list=($(eval echo '"${lv_config_name_list["'{0..$(($multires_lv_count - 2))}'"]}"')) + +resource_set_globals + +marsadm_timeout=10 + +# lvs lv-8-700 for all hosts +eval cluster_mars_dir_lv_name_list=($(x=(${main_host_list[@]/#/[}); y=(${x[@]/%/]=lv-$multires_lv_count-$multires_lv_size}); echo ${y[@]})) + +verbose_script=1 + +resource_recreate_fs_on_data_device_required=0 + +run_list="multires_prepare multires_run" diff --git a/test_suite/test_cases/multiresource/out/out.conf b/test_suite/test_cases/multiresource/out/out.conf new file mode 100644 index 00000000..84f4b84f --- /dev/null +++ b/test_suite/test_cases/multiresource/out/out.conf @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +# lvs lv-1-700 ... lv-8-700 +lv_config_name_list=($(eval echo lv-{1..$multires_lv_count}-$multires_lv_size)) + +# lvs lv-1-700 ... lv-7-700 +resource_name_list=($(eval echo '"${lv_config_name_list["'{0..$(($multires_lv_count - 2))}'"]}"')) + +resource_set_globals + +marsadm_timeout=10 + +# lvs lv-8-700 for all hosts +eval cluster_mars_dir_lv_name_list=($(x=(${main_host_list[@]/#/[}); y=(${x[@]/%/]=lv-$multires_lv_count-$multires_lv_size}); echo ${y[@]})) + +verbose_script=1 + +resource_recreate_fs_on_data_device_required=0 + +run_list="multires_prepare multires_run" diff --git a/test_suite/test_cases/perf.tmp/sync/sync.conf b/test_suite/test_cases/perf.tmp/sync/sync.conf deleted file mode 100644 index 763a35bf..00000000 --- a/test_suite/test_cases/perf.tmp/sync/sync.conf +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG -# -# Email: frank.liepold@1und1.de -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -verbose_script=1 - -run_list="resource_prepare resource_run synctest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf/apply/apply.conf b/test_suite/test_cases/perf/apply/apply.conf new file mode 100644 index 00000000..a925c36f --- /dev/null +++ b/test_suite/test_cases/perf/apply/apply.conf @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +perftest_action='apply' + +## time for which the amount of data to apply must be constant to declare +## the apply process as having stopped +perftest_time_constant_apply=10 + +## maxtime to wait for apply to stop (after pause-apply) +perftest_maxtime_apply=300 + +resource_recreate_fs_on_data_device_required=0 + +run_list="perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf.tmp/sync/no_parallel_writer.conf b/test_suite/test_cases/perf/apply/no_parallel_writer.conf similarity index 97% rename from test_suite/test_cases/perf.tmp/sync/no_parallel_writer.conf rename to test_suite/test_cases/perf/apply/no_parallel_writer.conf index 1e413702..33e67a65 100644 --- a/test_suite/test_cases/perf.tmp/sync/no_parallel_writer.conf +++ b/test_suite/test_cases/perf/apply/no_parallel_writer.conf @@ -21,5 +21,5 @@ verbose_script=1 -synctest_parallel_writer=0 +perftest_parallel_writer=0 diff --git a/test_suite/test_cases/perf.tmp/sync/parallel_writer.conf b/test_suite/test_cases/perf/apply/parallel_writer.conf similarity index 90% rename from test_suite/test_cases/perf.tmp/sync/parallel_writer.conf rename to test_suite/test_cases/perf/apply/parallel_writer.conf index 3b7eeb8e..a0d431d6 100644 --- a/test_suite/test_cases/perf.tmp/sync/parallel_writer.conf +++ b/test_suite/test_cases/perf/apply/parallel_writer.conf @@ -20,6 +20,6 @@ ##################################################################### verbose_script=1 -synctest_parallel_writer=1 -run_list="synctest_run lib_err_check_and_move_global_err_files_all" +perftest_parallel_writer=1 +resource_recreate_fs_on_data_device_required=1 diff --git a/test_suite/test_cases/perf/division_mars_device_data_device.ignore/division_mars_device_data_device.conf b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/division_mars_device_data_device.conf new file mode 100644 index 00000000..bf5ea626 --- /dev/null +++ b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/division_mars_device_data_device.conf @@ -0,0 +1,39 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +perftest_action='write' + +## hosts the tests are running on. The first host is initially always used as +## primary host +main_host_list=("istore-test-bap7" "istore-test-bs7") + +## time to run the data device writer to determine write rate +perftest_write_time=60 + +perftest_result_type="loops_per_min" + +perftest_division_mars_device_data_device=1 + +lib_rw_part_of_device_size_written_per_loop=8 + +resource_recreate_fs_on_data_device_required=1 + +run_list="perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf.tmp/sync/no_fast_sync.conf b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/same_controller.conf similarity index 91% rename from test_suite/test_cases/perf.tmp/sync/no_fast_sync.conf rename to test_suite/test_cases/perf/division_mars_device_data_device.ignore/same_controller.conf index c7334a2d..d76adc17 100644 --- a/test_suite/test_cases/perf.tmp/sync/no_fast_sync.conf +++ b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/same_controller.conf @@ -21,6 +21,5 @@ verbose_script=1 -synctest_fast_sync=0 +perftest_device_division="same_controller" -run_list="synctest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller.conf b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller.conf new file mode 100644 index 00000000..0d97842d --- /dev/null +++ b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller.conf @@ -0,0 +1,49 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +verbose_script=1 + +## this device must use physical volumes on a raid controller with the following +## properties: +## - different from the controller of the data device mount_mount_data_device +## - bbu cache can be switched on and off +## on the primary host a 18G lv can be created by: +## - pvcreate /dev/ +## - vgextend /dev/ +## - lvcreate -n lv-7-18 -L 18G /dev/ + +perftest_lv_on_bbu_controller="lv-7-18" + +perftest_dev_on_bbu_controller_list=(\ + [istore-test-bap7]=/dev/cciss/c0d0p7\ + ) + +cluster_mars_dir_lv_name_list["${main_host_list[0]}"]="$perftest_lv_on_bbu_controller" + +perftest_bbu_disable_cmd_list=(\ + "hpacucli ctrl slot=0 modify nobatterywritecache=disable"\ + ) +perftest_bbu_enable_cmd_list=(\ + "hpacucli ctrl slot=0 modify nobatterywritecache=enable"\ + "hpacucli ctrl slot=0 modify cacheratio=0/100"\ + ) +perftest_bbu_show_cmd_list=("hpacucli ctrl slot=0 show") + diff --git a/test_suite/test_cases/perf.tmp/sync/fast_sync.conf b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller/mars_dev_with_bbu.conf similarity index 91% rename from test_suite/test_cases/perf.tmp/sync/fast_sync.conf rename to test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller/mars_dev_with_bbu.conf index 9167891a..af6e8fee 100644 --- a/test_suite/test_cases/perf.tmp/sync/fast_sync.conf +++ b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller/mars_dev_with_bbu.conf @@ -21,6 +21,5 @@ verbose_script=1 -synctest_fast_sync=1 +perftest_device_division="separated_and_mars_dev_with_bbu_cache" -run_list="synctest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf.tmp/sync/rsync.conf b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller/mars_dev_without_bbu.conf similarity index 90% rename from test_suite/test_cases/perf.tmp/sync/rsync.conf rename to test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller/mars_dev_without_bbu.conf index e270c5c7..2773e078 100644 --- a/test_suite/test_cases/perf.tmp/sync/rsync.conf +++ b/test_suite/test_cases/perf/division_mars_device_data_device.ignore/separate_controller/mars_dev_without_bbu.conf @@ -21,7 +21,5 @@ verbose_script=1 -synctest_use_mars_sync=0 - -run_list="synctest_run lib_err_check_and_move_global_err_files_all" +perftest_device_division="separated_and_mars_dev_without_bbu_cache" diff --git a/test_suite/test_cases/perf/fetch/fetch.conf b/test_suite/test_cases/perf/fetch/fetch.conf new file mode 100644 index 00000000..849d131a --- /dev/null +++ b/test_suite/test_cases/perf/fetch/fetch.conf @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +perftest_action='fetch' + +## time for which the amount of data to apply must be constant to declare +## the apply process as having stopped +perftest_time_constant_apply=10 + +## maxtime to wait for apply to stop (after pause-apply) +perftest_maxtime_apply=300 + +resource_recreate_fs_on_data_device_required=0 + + +run_list="perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf/fetch/no_parallel_writer.conf b/test_suite/test_cases/perf/fetch/no_parallel_writer.conf new file mode 100644 index 00000000..33e67a65 --- /dev/null +++ b/test_suite/test_cases/perf/fetch/no_parallel_writer.conf @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +verbose_script=1 + +perftest_parallel_writer=0 + diff --git a/test_suite/test_cases/perf/fetch/parallel_writer.conf b/test_suite/test_cases/perf/fetch/parallel_writer.conf new file mode 100644 index 00000000..a0d431d6 --- /dev/null +++ b/test_suite/test_cases/perf/fetch/parallel_writer.conf @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +verbose_script=1 +perftest_parallel_writer=1 +resource_recreate_fs_on_data_device_required=1 + diff --git a/test_suite/test_cases/perf/fetch_and_apply/fetch_and_apply.conf b/test_suite/test_cases/perf/fetch_and_apply/fetch_and_apply.conf new file mode 100644 index 00000000..def0925f --- /dev/null +++ b/test_suite/test_cases/perf/fetch_and_apply/fetch_and_apply.conf @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +perftest_action='fetch_and_apply' + +## amount of data in GB to be written to the data device +perftest_data_in_gb_to_write=5 + +## size of logfile (in GiB and rounded) after perftest_data_in_gb_to_write are written +perftest_logfile_size_in_gb=5 + + +## apply should be much faster than fetch, so we wait only 2 seconds for apply to stop +perftest_wait_for_apply_to_stop_after_fetch_end=2 + +## time for which the amount of data to apply must be constant to declare +## the apply process as having stopped +perftest_time_constant_apply=10 + +## maxtime to wait for apply to stop (after pause-apply) +perftest_maxtime_apply=300 + +resource_recreate_fs_on_data_device_required=0 + + +run_list="perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf/fetch_and_apply/no_parallel_writer.conf b/test_suite/test_cases/perf/fetch_and_apply/no_parallel_writer.conf new file mode 100644 index 00000000..33e67a65 --- /dev/null +++ b/test_suite/test_cases/perf/fetch_and_apply/no_parallel_writer.conf @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +verbose_script=1 + +perftest_parallel_writer=0 + diff --git a/test_suite/test_cases/perf/fetch_and_apply/parallel_writer.conf b/test_suite/test_cases/perf/fetch_and_apply/parallel_writer.conf new file mode 100644 index 00000000..a0d431d6 --- /dev/null +++ b/test_suite/test_cases/perf/fetch_and_apply/parallel_writer.conf @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +##################################################################### + +verbose_script=1 +perftest_parallel_writer=1 +resource_recreate_fs_on_data_device_required=1 + diff --git a/test_suite/test_cases/perf/sync/fast_sync.conf b/test_suite/test_cases/perf/sync/fast_sync.conf index 3a2e1099..27821baa 100644 --- a/test_suite/test_cases/perf/sync/fast_sync.conf +++ b/test_suite/test_cases/perf/sync/fast_sync.conf @@ -21,4 +21,4 @@ verbose_script=1 -synctest_fast_sync=1 +perftest_sync_mode="fast_sync" diff --git a/test_suite/test_cases/perf/sync/no_fast_sync.conf b/test_suite/test_cases/perf/sync/no_fast_sync.conf index 602ae0c3..c4d03a62 100644 --- a/test_suite/test_cases/perf/sync/no_fast_sync.conf +++ b/test_suite/test_cases/perf/sync/no_fast_sync.conf @@ -21,5 +21,5 @@ verbose_script=1 -synctest_fast_sync=0 +perftest_sync_mode="no_fast_sync" diff --git a/test_suite/test_cases/perf/sync/no_parallel_writer.conf b/test_suite/test_cases/perf/sync/no_parallel_writer.conf index 1e413702..33e67a65 100644 --- a/test_suite/test_cases/perf/sync/no_parallel_writer.conf +++ b/test_suite/test_cases/perf/sync/no_parallel_writer.conf @@ -21,5 +21,5 @@ verbose_script=1 -synctest_parallel_writer=0 +perftest_parallel_writer=0 diff --git a/test_suite/test_cases/perf/sync/parallel_writer.conf b/test_suite/test_cases/perf/sync/parallel_writer.conf index 693a1b93..a0d431d6 100644 --- a/test_suite/test_cases/perf/sync/parallel_writer.conf +++ b/test_suite/test_cases/perf/sync/parallel_writer.conf @@ -20,4 +20,6 @@ ##################################################################### verbose_script=1 -synctest_parallel_writer=1 +perftest_parallel_writer=1 +resource_recreate_fs_on_data_device_required=1 + diff --git a/test_suite/test_cases/perf/sync/rsync.conf b/test_suite/test_cases/perf/sync/rsync.conf index 8612e89e..09904699 100644 --- a/test_suite/test_cases/perf/sync/rsync.conf +++ b/test_suite/test_cases/perf/sync/rsync.conf @@ -21,6 +21,5 @@ verbose_script=1 -synctest_use_mars_sync=0 - +perftest_sync_mode="rsync" diff --git a/test_suite/test_cases/perf/sync/sync.conf b/test_suite/test_cases/perf/sync/sync.conf index 763a35bf..cf939360 100644 --- a/test_suite/test_cases/perf/sync/sync.conf +++ b/test_suite/test_cases/perf/sync/sync.conf @@ -19,4 +19,22 @@ verbose_script=1 -run_list="resource_prepare resource_run synctest_run lib_err_check_and_move_global_err_files_all" +perftest_action='sync' + +## number of patches the data device will be patched with +perftest_number_of_patches_list=(10 10 4 2) + +## list of lengths of the patch (in KB) the data device will be patched with +## must correspond to perftest_number_of_patches_list +perftest_patch_length_list=(1 50000 500000 1000000) + + +## proc file to switch sync modus +perftest_sync_mode_proc_file='/proc/sys/mars/do_fast_fullsync' + +## data file to user for rsync +perftest_data_file=$main_mars_directory/file_to_sync + +resource_recreate_fs_on_data_device_required=0 + +run_list="resource_prepare resource_run perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf/write/out/out.conf b/test_suite/test_cases/perf/write/out/out.conf new file mode 100644 index 00000000..df5ed554 --- /dev/null +++ b/test_suite/test_cases/perf/write/out/out.conf @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +perftest_action='write' + +## time to run the data device writer to determine write rate +perftest_write_time=30 + +perftest_result_type="loops_per_min" + +resource_recreate_fs_on_data_device_required=1 + +run_list="perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_cases/perf/write/write.conf b/test_suite/test_cases/perf/write/write.conf new file mode 100644 index 00000000..df5ed554 --- /dev/null +++ b/test_suite/test_cases/perf/write/write.conf @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2010-2013 Frank Liepold / 1&1 Internet AG +# +# Email: frank.liepold@1und1.de +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +verbose_script=1 + +perftest_action='write' + +## time to run the data device writer to determine write rate +perftest_write_time=30 + +perftest_result_type="loops_per_min" + +resource_recreate_fs_on_data_device_required=1 + +run_list="perftest_run lib_err_check_and_move_global_err_files_all" diff --git a/test_suite/test_framework.txt b/test_suite/test_framework.txt new file mode 100644 index 00000000..e11bf0c6 --- /dev/null +++ b/test_suite/test_framework.txt @@ -0,0 +1,35 @@ +read performance + +/tests/small_file/low_load +/tests/small_file/high_load +/tests/big_file/low_load +/tests/big_file/high_load + +/modules/build_env.sh + function create_big_file + +/default-build_env.conf + testfile=/tmp/bigfile + +/modules/read_perf.sh + function run_readperf + function ... + +/default-read_perf.conf + filesize=20 + no_threads=5 + +/tests/small_file.conf + filesize=10 + +/tests/big_file.conf + filesize=10000 + +/tests/low_load.conf + no_threads=40 + run_list="run_readperf" + +/tests/big_file/high_load/small_buf +/tests/small_buf.conf + bufsize=4 + diff --git a/test_suite/test_suite.txt b/test_suite/test_suite.txt index 1fb9e496..bce72d02 100644 --- a/test_suite/test_suite.txt +++ b/test_suite/test_suite.txt @@ -188,3 +188,25 @@ hardcore H1 gap_in_log create and repair H1.1 - pause-apply on secondar hardcore H3 late_log_comp belatedly completed H3.1 - pause-apply on secondary logfile after new logfile has already +--------------------------------------------------------------------------------------------------------------------- +perf P2 perf_general fetch only P2.1 - pause-replay - time + rate + apply only P2.2 - disconnect " + sync P2.3 - invalidate " + fetch & apply P2.4 - = normal mode - time + rate + - time + rate + per action + write P2.5 - writing data device - write rate + P2.6 equal to P2.1 to P2.4 as P2.1 to P2.4 + ... with parallel writing write rate + P2.9 + with n devices P2.1 to P2.9 with + 2 4 8 devices + + + + + + + + + diff --git a/test_suite/test_suite.txt.official b/test_suite/test_suite.txt.official new file mode 100644 index 00000000..3fb4c3b2 --- /dev/null +++ b/test_suite/test_suite.txt.official @@ -0,0 +1,194 @@ +# test_suite.txt.official Version 0.01 +# +# description of the tests to execute before a mars release +# +# author: Frank Liepold frank.liepold@1und1.de +# +# Can be printed with a2ps -R --rows=1 --columns=1 -l 130 -L101 test_suite.txt + + +abbreviations: + data_dev_writer : process writing to data device on primary and producing a protocoll containing statistics + about runtime and written data. + device cksum : checking that cksum primary = cksum secondary + +recovery procedures : Some testcases cause more or less serious crashes or standstills (e.g. A4.1 below). If + there are documented repair strategies they will be tested, too. + +category id Prio testname description testcase and steps to check +done(%) +========================================================================================================================= +basic B 3 marsadm testing of pre and the scope of tests is specified by the +50% post conditions of documents resource_states.txt and + all marsadm cmds states_and_actions.txt and comprises at the + moment about 20 tests of the most important + marsadm commands by checking their pre and + post conditions + +------------------------------------------------------------------------------------------------------------------------- +basic B1 2 wait_role marsadm secondary B1.1 - marsadm secondary marsadm ROLE must +100% resp. primary - marsadm ROLE return secondary + may only return - ls /dev/mars/... ls must fail + with success after + the device has B1.2 - marsadm primary marsadm ROLE must + disappeared resp. - marsadm ROLE return primary + appeared - ls /dev/mars/... ls must succeed +------------------------------------------------------------------------------------------------------------------------- +admin A1 1 growing growing the data A1.1 - start data_dev_writer - device cksum +100% device in a running - lvresize on primary and secondary + mars connection - pause-sync on primary and secondary + - marsadm resize on primary + - resume-sync on primary and secondary + - wait for sync end + - stop data_dev_writer + - wait for fetch and apply end +------------------------------------------------------------------------------------------------------------------------ +admin A2 2 secon2prima host a: primary A2.1 - start data_dev_writer - device cksum +100% host b: secondary - marsadm primary on host b (must fail) + switch secondary -> - stop data_dev_writer + primary on host b - umount data device + - marsadm primary on host b +------------------------------------------------------------------------------------------------------------------------ +admin A3 2 apply_fetch indepedency of apply A3.1 - start data_dev_writer apply must run to +100% and fetch - pause-apply on secondary (nearly) end of + - pause-fetch on secondary fetched logfile + - resume-apply on secondary + + A3.2 - start data_dev_writer the whole logfile + - pause-apply on secondary must be fetched + - pause-fetch on secondary + - stop data_dev_writer + - resume-fetch on secondary +----------------------------------------------------------------------------------------------------------------------- +hardcore H2 2 mars_dir_full /mars full H2.1 running full because of logfiles device cksum +100% is regarded as an generated by data_dev_writer + admin error - start data_dev_writer + until /mars full + - rmmod mars on all cluster hosts + - resize /mars + - modprobe mars on all cluster hosts + - start second data_dev_writer + - stop all data_dev_writers + H2.2 running full because another process + is filling /mars + +----------------------------------------------------------------------------------------------------------------------- +admin A5 3 datadev_full data device full A5.1 - start data_dev_writer device cksum +100% - wait for data device full + see A1.1 +----------------------------------------------------------------------------------------------------------------------- +admin A6 2 logrotate looping logrotate A6.1 - start data_dev_writer - device cksum +100% - endless loop logrotate - impact of logfile + - stop loop after n minutes sizes on write + - stop data_dev_writer performance + - wait for fetch and apply end - impact of + logrotate + frequency on write + performance +----------------------------------------------------------------------------------------------------------------------- +admin A7 2 logdelete looping logrotate A7.1 - start data_dev_writer see A6.1 +100% and logdelete - endless loop logrotate + and logdelete + - stop loop after n minutes + - stop data_dev_writer + - wait for fetch and apply end +----------------------------------------------------------------------------------------------------------------------- +admin A8 3 compatibel compatibility of these testcases are to be implemented, when +0% mars versions there are different versions in production + userspace versions + kernel versions +----------------------------------------------------------------------------------------------------------------------- +admin A9 3 standstill recognizing, The most important part is done by marsview +80% indicating and + repair of A9.1 - logfile damage on secondary - error indicator + exceptional (still to specify) + standstills - repair (if + automatable) + - device cksum +----------------------------------------------------------------------------------------------------------------------- +admin A10 3 mult_device multiple data A10.* run several tests parallel - given by the single +0% devices (resources) on multiple mars connections where tests + per host the data devices are in some cases - impact on write + located on the same host performance + still to specify + A10.1 - for i in 1 2 3; do + start data_dev_writer on $i resources + stop data_dev_writer on resources + take write rate of each resource + A10.2 - like A10.1 but with regular log-rotate + and log-delete + +------------------------------------------------------------------------------------------------------------------------ +admin A11 3 small_sec_dev secondary data At the moment the two devices must have the +0% device smaller at cmd same size + marsadm join-resource + see mail uli 06/23/13 A11.1 - primary create resource (100 MB) - device cksum + - secondary join resource (80 MB) + - start data_dev_writer + - stop data_dev_writer + - switch primary -> secondary + - wait for fetch and apply end + on secondary +----------------------------------------------------------------------------------------------------------------------- +admin A12 3 casc_resize cascades of resize A12.1 to specify amount of synced +0% operations data +----------------------------------------------------------------------------------------------------------------------- +admin A13 3 sync_pos testing new symlink A13.1 to specify +0% syncpos +------------------------------------------------------------------------------------------------------------------------ +admin A14 3 filesys all tests on different +0% filesystems +----------------------------------------------------------------------------------------------------------------------- +perf P1 1 fullsync performance P1.1 - data on both data devices nearly - device cksum +100% matching (= secondary data device - sync time + patched with some bytes at some - transfer rate + offsets) + - default mars sync (fast fullsync) + - 2 GB data device + - secondary down + - secondary invalidate + - secondary up + - wait for sync end + P1.2 - similar to P1.1 but: + - "slow" mars sync + P1.3 - similar to P1.1 but: + - data on both with strong differences + P1.4 - similar to P1.1 but: + - data on both with strong differences + - "slow" mars sync + P1.11 equal to P1.1 - P1.4 but with - see P1.1 + ... data_dev_writer - impact of sync + P1.14 on write + performance +------------------------------------------------------------------------------------------------------------------------ +stabil S1 2 net_failure network broken S1.1 - start data_dev_writer - device cksum +S1.1: 100% - manipulation=total cut of connection - impact on write +S1.2ff: 0% - restore network connection performance + - stop data_dev_writer + S1.2ff similar to S1.1 but with different + network connection manipulations + still to specify +------------------------------------------------------------------------------------------------------------------------ +stabil S2 1 crash_prim reboot of primary S2.1 - start data_dev_writer device cksum +100% while writing - reboot primary (ipmitool) +------------------------------------------------------------------------------------------------------------------------ +stabil S3 3 crash_sec reboot of secondary S3.1 - start data_dev_writer device cksum +0% while applying and - reboot secondary + fetching +------------------------------------------------------------------------------------------------------------------------ +hardcore H1 3 gap_in_log create and repair H1.1 - pause-apply on secondary +H1.1: 100% gap in logfile - start data_dev_writer +H1.2: 0% - stop data_dev_writer after n minutes + - wait until fetch complete + - create gap at the end of last logfile + - resume-apply + - wait until apply stops apply must stop + at gap + - repair gap (apply must continue) device cksum + H1.2 - similar to H1.1 but gap in the middle + of the logfiles +------------------------------------------------------------------------------------------------------------------------- +hardcore H3 3 late_log_comp belatedly completed H3.1 to specify +0% logfile after new + logfile has already + arrived