#!/bin/bash -x set -e set -o functrace PS4=' ${FUNCNAME[0]}: $LINENO: ' function get_pg() { local pool obj map_output pg pool=$1 obj=$2 declare -a map_output map_output=($(ceph osd map $1 $2)) for (( i=0; i<${#map_output[*]}; i++ )) ; do if [ "${map_output[$i]}" == "pg" ] ; then pg=${map_output[((i+2))]} break fi done pg=$(echo $pg | sed 's/[()]//g') echo $pg } function expect_false() { set -x if "$@"; then return 1; else return 0; fi } TMPDIR=/tmp/cephtool$$ mkdir $TMPDIR trap "rm -fr $TMPDIR" 0 TMPFILE=$TMPDIR/test_invalid.$$ function check_response() { expected_stderr_string=$1 retcode=$2 expected_retcode=$3 if [ "$expected_retcode" -a $retcode != $expected_retcode ] ; then echo "return code invalid: got $retcode, expected $expected_retcode" >&2 exit 1 fi if ! grep "$expected_stderr_string" $TMPFILE >/dev/null 2>&1 ; then echo "Didn't find $expected_stderr_string in stderr output" >&2 echo "Stderr: " >&2 cat $TMPFILE >&2 exit 1 fi } function get_config_value_or_die() { local target config_opt raw val target=$1 config_opt=$2 raw="`ceph daemon $target config get $config_opt 2>/dev/null`" if [[ $? -ne 0 ]]; then echo "error obtaining config opt '$config_opt' from '$target': $raw" exit 1 fi raw=`echo $raw | sed -e 's/[{} "]//g'` val=`echo $raw | cut -f2 -d:` echo "$val" return 0 } function expect_config_value() { local target config_opt expected_val val target=$1 config_opt=$2 expected_val=$3 val=$(get_config_value_or_die $target $config_opt) if [[ "$val" != "$expected_val" ]]; then echo "expected '$expected_val', got '$val'" exit 1 fi } function test_mon_injectargs_SI() { # Test SI units during injectargs and 'config set' # We only aim at testing the units are parsed accordingly # and don't intend to test whether the options being set # actually expect SI units to be passed. # Keep in mind that all integer based options (i.e., INT, # LONG, U32, U64) will accept SI unit modifiers. initial_value=$(get_config_value_or_die "mon.a" "mon_pg_warn_min_objects") ceph daemon mon.a config set mon_pg_warn_min_objects 10 expect_config_value "mon.a" "mon_pg_warn_min_objects" 10 ceph daemon mon.a config set mon_pg_warn_min_objects 10K expect_config_value "mon.a" "mon_pg_warn_min_objects" 10240 ceph daemon mon.a config set mon_pg_warn_min_objects 1G expect_config_value "mon.a" "mon_pg_warn_min_objects" 1073741824 ceph daemon mon.a config set mon_pg_warn_min_objects 10F > $TMPFILE || true check_response "'10F': (22) Invalid argument" # now test with injectargs ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10' expect_config_value "mon.a" "mon_pg_warn_min_objects" 10 ceph tell mon.a injectargs '--mon_pg_warn_min_objects 10K' expect_config_value "mon.a" "mon_pg_warn_min_objects" 10240 ceph tell mon.a injectargs '--mon_pg_warn_min_objects 1G' expect_config_value "mon.a" "mon_pg_warn_min_objects" 1073741824 expect_false ceph injectargs mon.a '--mon_pg_warn_min_objects 10F' ceph daemon mon.a config set mon_pg_warn_min_objects $initial_value } function test_tiering() { # tiering ceph osd pool create cache 2 ceph osd pool create cache2 2 ceph osd tier add data cache ceph osd tier add data cache2 expect_false ceph osd tier add metadata cache # test some state transitions ceph osd tier cache-mode cache writeback ceph osd tier cache-mode cache forward ceph osd tier cache-mode cache readonly ceph osd tier cache-mode cache forward ceph osd tier cache-mode cache none ceph osd tier cache-mode cache writeback expect_false ceph osd tier cache-mode cache none expect_false ceph osd tier cache-mode cache readonly # test with dirty objects in the tier pool # tier pool currently set to 'writeback' rados -p cache put /etc/passwd /etc/passwd ceph tell osd.* flush_pg_stats || true # 1 dirty object in pool 'cache' ceph osd tier cache-mode cache forward expect_false ceph osd tier cache-mode cache none expect_false ceph osd tier cache-mode cache readonly ceph osd tier cache-mode cache writeback # remove object from tier pool rados -p cache rm /etc/passwd rados -p cache cache-flush-evict-all ceph tell osd.* flush_pg_stats || true # no dirty objects in pool 'cache' ceph osd tier cache-mode cache forward ceph osd tier cache-mode cache none ceph osd tier cache-mode cache readonly TRIES=0 while ! ceph osd pool set cache pg_num 3 --yes-i-really-mean-it 2>$TMPFILE do grep 'currently creating pgs' $TMPFILE TRIES=$(( $TRIES + 1 )) test $TRIES -ne 60 sleep 3 done expect_false ceph osd pool set cache pg_num 4 ceph osd tier cache-mode cache none ceph osd tier set-overlay data cache expect_false ceph osd tier set-overlay data cache2 expect_false ceph osd tier remove data cache ceph osd tier remove-overlay data ceph osd tier set-overlay data cache2 ceph osd tier remove-overlay data ceph osd tier remove data cache ceph osd tier add metadata cache expect_false ceph osd tier set-overlay data cache ceph osd tier set-overlay metadata cache ceph osd tier remove-overlay metadata ceph osd tier remove metadata cache ceph osd tier remove data cache2 # make sure a non-empty pool fails rados -p cache2 put /etc/passwd /etc/passwd while ! ceph df | grep cache2 | grep ' 1 ' ; do echo waiting for pg stats to flush sleep 2 done expect_false ceph osd tier add data cache2 ceph osd tier add data cache2 --force-nonempty ceph osd tier remove data cache2 ceph osd pool delete cache cache --yes-i-really-really-mean-it ceph osd pool delete cache2 cache2 --yes-i-really-really-mean-it # convenient add-cache command ceph osd pool create cache3 2 ceph osd tier add-cache data cache3 1024000 ceph osd dump | grep cache3 | grep bloom | grep 'false_positive_probability: 0.05' | grep 'target_bytes 1024000' | grep '1200s x4' ceph osd tier remove data cache3 ceph osd pool delete cache3 cache3 --yes-i-really-really-mean-it # protection against pool removal when used as tiers ceph osd pool create datapool 2 ceph osd pool create cachepool 2 ceph osd tier add-cache datapool cachepool 1024000 ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it 2> $TMPFILE || true check_response "EBUSY: pool 'cachepool' is a tier of 'datapool'" ceph osd pool delete datapool datapool --yes-i-really-really-mean-it 2> $TMPFILE || true check_response "EBUSY: pool 'datapool' has tiers cachepool" ceph osd tier remove datapool cachepool ceph osd pool delete cachepool cachepool --yes-i-really-really-mean-it ceph osd pool delete datapool datapool --yes-i-really-really-mean-it # check health check ceph osd pool create cache4 2 ceph osd pool set cache4 target_max_objects 5 ceph osd pool set cache4 target_max_bytes 1000 for f in `seq 1 5` ; do rados -p cache4 put foo$f /etc/passwd done while ! ceph df | grep cache4 | grep ' 5 ' ; do echo waiting for pg stats to flush sleep 2 done ceph health | grep WARN | grep cache4 ceph health detail | grep cache4 | grep 'target max' | grep objects ceph health detail | grep cache4 | grep 'target max' | grep 'B' ceph osd pool delete cache4 cache4 --yes-i-really-really-mean-it } function test_auth() { ceph auth add client.xx mon allow osd "allow *" ceph auth export client.xx >client.xx.keyring ceph auth add client.xx -i client.xx.keyring rm -f client.xx.keyring ceph auth list | grep client.xx ceph auth get client.xx | grep caps | grep mon ceph auth get client.xx | grep caps | grep osd ceph auth get-key client.xx ceph auth print-key client.xx ceph auth print_key client.xx ceph auth caps client.xx osd "allow rw" expect_false "ceph auth get client.xx | grep caps | grep mon" ceph auth get client.xx | grep osd | grep "allow rw" ceph auth export | grep client.xx ceph auth export -o authfile ceph auth import -i authfile ceph auth export -o authfile2 diff authfile authfile2 rm authfile authfile2 ceph auth del client.xx } function test_mon_misc() { # with and without verbosity ceph osd dump | grep '^epoch' ceph --concise osd dump | grep '^epoch' # df ceph df > $TMPFILE grep GLOBAL $TMPFILE grep -v DIRTY $TMPFILE ceph df detail > $TMPFILE grep CATEGORY $TMPFILE grep DIRTY $TMPFILE ceph df --format json > $TMPFILE grep 'total_space' $TMPFILE grep -v 'dirty' $TMPFILE ceph df detail --format json > $TMPFILE grep 'rd_kb' $TMPFILE grep 'dirty' $TMPFILE ceph df --format xml | grep '' ceph df detail --format xml | grep '' ceph fsid ceph health ceph health detail ceph health --format json-pretty ceph health detail --format xml-pretty ceph -w > $TMPDIR/$$ & wpid="$!" mymsg="this is a test log message $$.$(date)" ceph log "$mymsg" sleep 3 if ! grep "$mymsg" $TMPDIR/$$; then # in case it is very slow (mon thrashing or something) sleep 30 grep "$mymsg" $TMPDIR/$$ fi kill $wpid } function test_mon_mds() { ceph mds cluster_down ceph mds cluster_up ceph mds compat rm_incompat 4 ceph mds compat rm_incompat 4 ceph mds compat show expect_false ceph mds deactivate 2 ceph mds dump # XXX mds fail, but how do you undo it? mdsmapfile=$TMPDIR/mdsmap.$$ current_epoch=$(ceph mds getmap -o $mdsmapfile --no-log-to-stderr 2>&1 | grep epoch | sed 's/.*epoch //') [ -s $mdsmapfile ] ((epoch = current_epoch + 1)) ceph mds setmap -i $mdsmapfile $epoch rm $mdsmapfile ceph mds newfs 0 1 --yes-i-really-mean-it ceph osd pool create data2 10 poolnum=$(ceph osd dump | grep 'pool.*data2' | awk '{print $2;}') ceph mds add_data_pool $poolnum ceph mds add_data_pool rbd ceph mds remove_data_pool $poolnum ceph mds remove_data_pool rbd ceph osd pool delete data2 data2 --yes-i-really-really-mean-it ceph mds set_max_mds 4 ceph mds set_max_mds 3 ceph mds set max_mds 4 expect_false ceph mds set max_mds asdf expect_false ceph mds set inline_data true ceph mds set inline_data true --yes-i-really-mean-it ceph mds set inline_data yes --yes-i-really-mean-it ceph mds set inline_data 1 --yes-i-really-mean-it expect_false ceph mds set inline_data --yes-i-really-mean-it ceph mds set inline_data false ceph mds set inline_data no ceph mds set inline_data 0 expect_false ceph mds set inline_data asdf ceph mds set max_file_size 1048576 expect_false ceph mds set max_file_size 123asdf expect_false ceph mds set allow_new_snaps expect_false ceph mds set allow_new_snaps true ceph mds set allow_new_snaps true --yes-i-really-mean-it ceph mds set allow_new_snaps 0 ceph mds set allow_new_snaps false ceph mds set allow_new_snaps no expect_false ceph mds set allow_new_snaps taco # we should never be able to add EC pools as data or metadata pools # create an ec-pool ceph osd pool create mds-ec-pool 10 10 erasure set +e ceph mds add_data_pool mds-ec-pool 2>$TMPFILE check_response 'erasure-code' $? 22 set -e poolnum=$(ceph osd dump | grep 'pool.*mds-ec-pool' | awk '{print $2;}') set +e ceph mds newfs 0 $poolnum --yes-i-really-mean-it 2>$TMPFILE check_response 'erasure-code' $? 22 ceph mds newfs $poolnum 1 --yes-i-really-mean-it 2>$TMPFILE check_response 'erasure-code' $? 22 ceph mds newfs $poolnum $poolnum --yes-i-really-mean-it 2>$TMPFILE check_response 'erasure-code' $? 22 set -e ceph osd pool delete mds-ec-pool mds-ec-pool --yes-i-really-really-mean-it ceph mds stat # ceph mds tell mds.a getmap # ceph mds rm # ceph mds rmfailed # ceph mds set_state # ceph mds stop } function test_mon_mon() { # no mon add/remove ceph mon dump ceph mon getmap -o $TMPDIR/monmap.$$ [ -s $TMPDIR/monmap.$$ ] # ceph mon tell ceph mon_status } function test_mon_osd() { # # osd blacklist # bl=192.168.0.1:0/1000 ceph osd blacklist add $bl ceph osd blacklist ls | grep $bl ceph osd blacklist rm $bl expect_false "ceph osd blacklist ls | grep $bl" bl=192.168.0.1 # test without nonce, invalid nonce ceph osd blacklist add $bl ceph osd blacklist ls | grep $bl ceph osd blacklist rm $bl expect_false "ceph osd blacklist ls | grep $bl" expect_false "ceph osd blacklist $bl/-1" expect_false "ceph osd blacklist $bl/foo" # # osd crush # ceph osd crush tunables legacy ceph osd crush show-tunables | grep argonaut ceph osd crush tunables bobtail ceph osd crush show-tunables | grep bobtail ceph osd crush tunables firefly ceph osd crush show-tunables | grep firefly # # osd scrub # # how do I tell when these are done? ceph osd scrub 0 ceph osd deep-scrub 0 ceph osd repair 0 for f in noup nodown noin noout noscrub nodeep-scrub nobackfill norecover notieragent do ceph osd set $f ceph osd unset $f done expect_false ceph osd set bogus expect_false ceph osd unset bogus ceph osd set noup ceph osd down 0 ceph osd dump | grep 'osd.0 down' ceph osd unset noup for ((i=0; i < 100; i++)); do if ! ceph osd dump | grep 'osd.0 up'; then echo "waiting for osd.0 to come back up" sleep 10 else break fi done ceph osd thrash 10 ceph osd down `seq 0 31` # force everything down so that we can trust up # make sure everything gets back up+in. for ((i=0; i < 100; i++)); do if ceph osd dump | grep ' down '; then echo "waiting for osd(s) to come back up" sleep 10 else break fi done # if you have more osds than this you are on your own for f in `seq 0 31`; do ceph osd in $f || true done ceph osd dump | grep 'osd.0 up' ceph osd find 1 ceph osd metadata 1 | grep 'distro' ceph osd out 0 ceph osd dump | grep 'osd.0.*out' ceph osd in 0 ceph osd dump | grep 'osd.0.*in' ceph osd find 0 f=$TMPDIR/map.$$ ceph osd getcrushmap -o $f [ -s $f ] rm $f ceph osd getmap -o $f [ -s $f ] rm $f save=$(ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//') ceph osd setmaxosd 10 ceph osd getmaxosd | grep 'max_osd = 10' ceph osd setmaxosd $save ceph osd getmaxosd | grep "max_osd = $save" for id in `ceph osd ls` ; do ceph tell osd.$id version done ceph osd rm 0 2>&1 | grep 'EBUSY' id=`ceph osd create` ceph osd lost $id --yes-i-really-mean-it ceph osd rm $id uuid=`uuidgen` id=`ceph osd create $uuid` id2=`ceph osd create $uuid` [ "$id" = "$id2" ] ceph osd rm $id ceph osd ls ceph osd lspools | grep data ceph osd map data foo | grep 'pool.*data.*object.*foo.*pg.*up.*acting' ceph osd pause ceph osd dump | grep 'flags pauserd,pausewr' ceph osd unpause ceph osd tree ceph osd stat | grep up, } function test_mon_osd_pool() { # # osd pool # ceph osd pool mksnap data datasnap rados -p data lssnap | grep datasnap ceph osd pool rmsnap data datasnap ceph osd pool create data2 10 ceph osd pool rename data2 data3 ceph osd lspools | grep data3 ceph osd pool delete data3 data3 --yes-i-really-really-mean-it ceph osd pool create replicated 12 12 replicated ceph osd pool create replicated 12 12 replicated ceph osd pool create replicated 12 12 # default is replicated ceph osd pool create replicated 12 # default is replicated, pgp_num = pg_num # should fail because the type is not the same expect_false ceph osd pool create replicated 12 12 erasure ceph osd lspools | grep replicated ceph osd pool delete replicated replicated --yes-i-really-really-mean-it } function test_mon_pg() { ceph pg debug unfound_objects_exist ceph pg debug degraded_pgs_exist ceph pg deep-scrub 0.0 ceph pg dump ceph pg dump pgs_brief --format=json ceph pg dump pgs --format=json ceph pg dump pools --format=json ceph pg dump osds --format=json ceph pg dump sum --format=json ceph pg dump all --format=json ceph pg dump pgs_brief osds --format=json ceph pg dump pools osds pgs_brief --format=json ceph pg dump_json ceph pg dump_pools_json ceph pg dump_stuck inactive ceph pg dump_stuck unclean ceph pg dump_stuck stale # can't test this... # ceph pg force_create_pg ceph pg getmap -o $TMPDIR/map.$$ [ -s $TMPDIR/map.$$ ] ceph pg map 0.0 | grep acting ceph pg repair 0.0 ceph pg scrub 0.0 ceph pg send_pg_creates ceph pg set_full_ratio 0.90 ceph pg dump --format=plain | grep '^full_ratio 0.9' ceph pg set_full_ratio 0.95 ceph pg set_nearfull_ratio 0.90 ceph pg dump --format=plain | grep '^nearfull_ratio 0.9' ceph pg set_nearfull_ratio 0.85 ceph pg stat | grep 'pgs:' ceph pg 0.0 query ceph tell 0.0 query ceph quorum enter ceph quorum_status ceph report | grep osd_stats ceph status ceph -s # # tell osd version # ceph tell osd.0 version expect_false ceph tell osd.9999 version expect_false ceph tell osd.foo version # back to pg stuff ceph tell osd.0 dump_pg_recovery_stats | grep Started ceph osd reweight 0 0.9 expect_false ceph osd reweight 0 -1 ceph osd reweight 0 1 ceph osd primary-affinity osd.0 .9 expect_false ceph osd primary-affinity osd.0 -2 ceph osd primary-affinity osd.0 1 ceph osd pg-temp 0.0 0 1 2 ceph osd pg-temp 0.0 1 0 2 expect_false ceph osd pg-temp asdf qwer expect_false ceph osd pg-temp 0.0 asdf expect_false ceph osd pg-temp 0.0 # don't test ceph osd primary-temp for now } function test_mon_osd_pool_set() { for s in pg_num pgp_num size min_size crash_replay_interval crush_ruleset; do ceph osd pool get data $s done old_size=$(ceph osd pool get data size | sed -e 's/size: //') (( new_size = old_size + 1 )) ceph osd pool set data size $new_size ceph osd pool get data size | grep "size: $new_size" ceph osd pool set data size $old_size ceph osd pool create pool_erasure 12 12 erasure set +e ceph osd pool set pool_erasure size 4444 2>$TMPFILE check_response 'not change the size' set -e auid=5555 ceph osd pool set data auid $auid ceph osd pool get data auid | grep $auid ceph --format=xml osd pool get data auid | grep $auid ceph osd pool set data auid 0 ceph osd pool set data hashpspool true ceph osd pool set data hashpspool false ceph osd pool set data hashpspool 0 ceph osd pool set data hashpspool 1 expect_false ceph osd pool set data hashpspool asdf expect_false ceph osd pool set data hashpspool 2 ceph osd pool set rbd hit_set_type explicit_hash ceph osd pool get rbd hit_set_type | grep "hit_set_type: explicit_hash" ceph osd pool set rbd hit_set_type explicit_object ceph osd pool get rbd hit_set_type | grep "hit_set_type: explicit_object" ceph osd pool set rbd hit_set_type bloom ceph osd pool get rbd hit_set_type | grep "hit_set_type: bloom" expect_false ceph osd pool set rbd hit_set_type i_dont_exist ceph osd pool set rbd hit_set_period 123 ceph osd pool get rbd hit_set_period | grep "hit_set_period: 123" ceph osd pool set rbd hit_set_count 12 ceph osd pool get rbd hit_set_count | grep "hit_set_count: 12" ceph osd pool set rbd hit_set_fpp .01 ceph osd pool get rbd hit_set_fpp | grep "hit_set_fpp: 0.01" ceph osd pool set rbd target_max_objects 123 ceph osd pool set rbd target_max_bytes 123456 ceph osd pool set rbd cache_target_dirty_ratio .123 expect_false ceph osd pool set rbd cache_target_dirty_ratio -.2 expect_false ceph osd pool set rbd cache_target_dirty_ratio 1.1 ceph osd pool set rbd cache_target_full_ratio .123 ceph osd dump -f json-pretty | grep '"cache_target_full_ratio_micro": 123000' ceph osd pool set rbd cache_target_full_ratio 1.0 ceph osd pool set rbd cache_target_full_ratio 0 expect_false ceph osd pool set rbd cache_target_full_ratio 1.1 ceph osd pool set rbd cache_min_flush_age 123 ceph osd pool set rbd cache_min_evict_age 234 ceph osd pool get rbd crush_ruleset | grep 'crush_ruleset: 0' } function test_mon_osd_erasure_code() { ceph osd erasure-code-profile set fooprofile a=b c=d ceph osd erasure-code-profile set fooprofile a=b c=d expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f ceph osd erasure-code-profile set fooprofile a=b c=d e=f --force ceph osd erasure-code-profile set fooprofile a=b c=d e=f expect_false ceph osd erasure-code-profile set fooprofile a=b c=d e=f g=h # # cleanup by removing profile 'fooprofile' ceph osd erasure-code-profile rm fooprofile } function test_mon_osd_misc() { set +e # expect error about missing 'pool' argument ceph osd map 2>$TMPFILE; check_response 'pool' $? 22 # expect error about unused argument foo ceph osd ls foo 2>$TMPFILE; check_response 'unused' $? 22 # expect "not in range" for invalid full ratio ceph pg set_full_ratio 95 2>$TMPFILE; check_response 'not in range' $? 22 # expect "not in range" for invalid overload percentage ceph osd reweight-by-utilization 80 2>$TMPFILE; check_response 'not in range' $? 22 set -e } function test_mon_heap_profiler() { # expect 'heap' commands to be correctly parsed ceph heap stats ceph heap start_profiler ceph heap dump ceph heap stop_profiler ceph heap release } function test_osd_bench() { # test osd bench limits # As we should not rely on defaults (as they may change over time), # lets inject some values and perform some simple tests # max iops: 10 # 100 IOPS # max throughput: 10485760 # 10MB/s # max block size: 2097152 # 2MB # duration: 10 # 10 seconds ceph tell osd.0 injectargs "\ --osd-bench-duration 10 \ --osd-bench-max-block-size 2097152 \ --osd-bench-large-size-max-throughput 10485760 \ --osd-bench-small-size-max-iops 10" # anything with a bs larger than 2097152 must fail expect_false ceph tell osd.0 bench 1 2097153 # but using 'osd_bench_max_bs' must succeed ceph tell osd.0 bench 1 2097152 # we assume 1MB as a large bs; anything lower is a small bs # for a 4096 bytes bs, for 10 seconds, we are limited by IOPS # max count: 409600 # more than max count must not be allowed expect_false ceph tell osd.0 bench 409601 4096 # but 409600 must be succeed ceph tell osd.0 bench 409600 4096 # for a large bs, we are limited by throughput. # for a 2MB block size for 10 seconds, out max count is 50 # max count: 50 # more than max count must not be allowed expect_false ceph tell osd.0 bench 51 2097152 # but 50 must succeed ceph tell osd.0 bench 50 2097152 } # # New tests should be added to the TESTS array below # # Individual tests may be run using the '-t ' argument # The user can specify '-t ' as many times as she wants # # Tests will be run in order presented in the TESTS array, or in # the order specified by the '-t ' options. # # '-l' will list all the available test names # '-h' will show usage # # The test maintains backward compatibility: not specifying arguments # will run all tests following the order they appear in the TESTS array. # set +x TESTS=( mon_injectargs_SI tiering auth mon_misc mon_mds mon_mon mon_osd mon_osd_pool mon_pg mon_osd_pool_set mon_osd_erasure_code mon_osd_misc mon_heap_profiler osd_bench ) # # "main" follows # function list_tests() { echo "AVAILABLE TESTS" for i in ${TESTS[@]}; do echo " $i" done } function usage() { echo "usage: $0 [-h|-l|-t [-t ...]]" } tests_to_run=() while [[ $# -gt 0 ]]; do opt=$1 case "$opt" in "-l" ) do_list=1 ;; "-t" ) shift if [[ -z "$1" ]]; then echo "missing argument to '-t'" usage ; exit 1 fi tests_to_run=("${tests_to_run[@]}" "$1") ;; "-h" ) usage ; exit 0 ;; esac shift done if [[ $do_list -eq 1 ]]; then list_tests ; exit 0 fi if [[ ${#tests_to_run[@]} -eq 0 ]]; then tests_to_run=("${TESTS[@]}") fi for i in ${tests_to_run[@]}; do set -x test_${i} ; set +x done set -x echo OK