#!/bin/sh _help() { cat << EOF ### run-regtests.sh ### Running run-regtests.sh --help shows this information about how to use it Run without parameters to run all tests in the current folder (including subfolders) run-regtests.sh Provide paths to run tests from (including subfolders): run-regtests.sh ./tests1 ./tests2 Parameters: --j , To run vtest with multiple jobs / threads for a faster overall result run-regtests.sh ./fasttest --j 16 --v, to run verbose run-regtests.sh --v, disables the default vtest 'quiet' parameter --debug to show test logs on standard output (implies --v) run-regtests.sh --debug --keep-logs to keep all log directories (by default kept if test fails) run-regtests.sh --keep-logs --vtestparams , passes custom ARGS to vtest run-regtests.sh --vtestparams "-n 10" --type filter the types of the tests to be run, depending on the commented REGTESTS_TYPE variable value in each VTC file. The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel and experimental. When not specified, it is set to 'default' as default value. run-regtest.sh --type slow,default --clean to cleanup previous reg-tests log directories and exit run-regtests.sh --clean Including text below into a .vtc file will check for its requirements related to haproxy's target and compilation options # Below targets are not capable of completing this test successfully #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd #EXCLUDE_TARGETS=dos,freebsd,windows # Below option is required to complete this test successfully #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in. #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA #REQUIRE_SERVICE=prometheus-exporter #REQUIRE_SERVICES=prometheus-exporter,foo # To define a range of versions that a test can run with: #REQUIRE_VERSION=0.0 #REQUIRE_VERSION_BELOW=99.9 Configure environment variables to set the haproxy and vtest binaries to use setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy setenv VTEST_PROGRAM /usr/local/bin/vtest setenv HAPROXY_ARGS "-dM -de -m 50" or export HAPROXY_PROGRAM=/usr/local/sbin/haproxy export VTEST_PROGRAM=/usr/local/bin/vtest export HAPROXY_ARGS="-dM -de -m 50" EOF exit 0 } add_range_to_test_list() { level0="*.vtc" level1="h*.vtc" level2="s*.vtc" level3="l*.vtc" level4="b*.vtc" level5="k*.vtc" level6="e*.vtc" new_range=$(echo $1 | tr '-' ' ') non_digit=$(echo $new_range | grep '[^0-9 ]') if [ -n "$non_digit" ] ; then return fi if [ "$new_range" = "$1" ] ; then if [ $1 -gt 6 ] ; then return fi eval echo '$'level$1 return fi if [ -z "$new_range" ] ; then return fi list= for l in $(seq $new_range) ; do if [ -n "l" ] ; then if [ -z "$list" ] ; then list="$(eval echo '$'level${l})" else list="$list $(eval echo '$'level${l})" fi fi done echo $list } _startswith() { _str="$1" _sub="$2" echo "$_str" | grep "^$_sub" >/dev/null 2>&1 } _findtests() { set -f REGTESTS_TYPES="${REGTESTS_TYPES:-default,bug,devel,slow}" any_test=$(echo $REGTESTS_TYPES | grep -cw "any") for i in $( find "$1" -name *.vtc ); do skiptest= OLDIFS="$IFS"; IFS="$LINEFEED" set -- $(grep '^#[0-9A-Z_]*=' "$i") IFS="$OLDIFS" require_version=""; require_version_below=""; require_options=""; require_services=""; exclude_targets=""; regtest_type="" requiredoption=""; requiredservice=""; excludedtarget=""; while [ $# -gt 0 ]; do v="$1"; v="${v#*=}" case "$1" in "#REQUIRE_VERSION="*) require_version="$v" ;; "#REQUIRE_VERSION_BELOW="*) require_version_below="$v" ;; "#REQUIRE_OPTIONS="*) require_options="$v" ;; "#REQUIRE_SERVICES="*) require_services="$v" ;; "#EXCLUDE_TARGETS="*) exclude_targets="$v" ;; "#REGTEST_TYPE="*) regtest_type="$v" ;; "#REQUIRE_OPTION="*) requiredoption="${v%,*}" ;; "#REQUIRE_SERVICE="*) required_service="${v%,*}" ;; "#EXCLUDE_TARGET="*) excludedtarget="${v%,*}" ;; # Note: any new variable declared here must be initialized above. esac shift done if [ $any_test -ne 1 ] ; then if [ -z $regtest_type ] ; then regtest_type=default fi if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then echo " Skip $i because its type '"$regtest_type"' is excluded" skiptest=1 fi fi if [ -n "$requiredoption" ]; then require_options="$require_options,$requiredoption" fi if [ -n "$requiredservice" ]; then require_services="$require_services,$requiredservice" fi if [ -n "$excludedtarget" ]; then exclude_targets="$exclude_targets,$excludedtarget" fi IFS=","; set -- $require_options; IFS=$OLDIFS; require_options="$*" IFS=","; set -- $require_services; IFS=$OLDIFS; require_services="$*" IFS=","; set -- $exclude_targets; IFS=$OLDIFS; exclude_targets="$*" if [ -n "$require_version" ]; then if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then echo " Skip $i because option haproxy is version: $HAPROXY_VERSION" echo " REASON: this test requires at least version: $require_version" skiptest=1 fi fi if [ -n "$require_version_below" ]; then if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then echo " Skip $i because option haproxy is version: $HAPROXY_VERSION" echo " REASON: this test requires a version below: $require_version_below" skiptest=1 fi fi for excludedtarget in $exclude_targets; do if [ "$excludedtarget" = "$TARGET" ]; then echo " Skip $i because haproxy is compiled for the excluded target $TARGET" skiptest=1 fi done for requiredoption in $require_options; do IFS="|"; set -- $requiredoption; IFS=$OLDIFS; alternatives="$*" found= for alt in $alternatives; do if [ -z "${FEATURES_PATTERN##* +$alt *}" ]; then found=1; fi done if [ -z $found ]; then echo " Skip $i because haproxy is not compiled with the required option $requiredoption" skiptest=1 fi done for requiredservice in $require_services; do IFS="|"; set -- $requiredservice; IFS=$OLDIFS; alternatives="$*" found= for alt in $alternatives; do if [ -z "${SERVICES_PATTERN##* $alt *}" ]; then found=1; fi done if [ -z $found ]; then echo " Skip $i because haproxy is not compiled with the required service $requiredservice" skiptest=1 fi done if [ -z $skiptest ]; then echo " Add test: $i" testlist="$testlist $i" fi done } _cleanup() { DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null) if [ -z "${DIRS}" ]; then echo "No reg-tests log directory found" else echo "Cleanup following reg-tests log directories:" for d in ${DIRS}; do echo " o ${TESTDIR}/$d" done read -p "Continue (y/n)?" reply case "$reply" in y|Y) for d in ${DIRS}; do rm -r "${TESTDIR}/$d" done echo "done" exit 0 ;; *) echo "aborted" exit 1 ;; esac fi } _process() { while [ ${#} -gt 0 ]; do if _startswith "$1" "-"; then case "${1}" in --j) jobcount="$2" shift ;; --vtestparams) vtestparams="$2" shift ;; --v) verbose="" ;; --debug) verbose="" debug="-v" ;; --keep-logs) keep_logs="-L" ;; --type) REGTESTS_TYPES="$2" shift ;; --clean) _cleanup exit 0 ;; --help) _help ;; *) echo "Unknown parameter : $1" exit 1 ;; esac else REGTESTS="${REGTESTS} $1" fi shift 1 done } _version() { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }'; } HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}" HAPROXY_ARGS="${HAPROXY_ARGS--dM}" VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}" TESTDIR="${TMPDIR:-/tmp}" REGTESTS="" LINEFEED=" " jobcount="" verbose="-q" debug="" keep_logs="-l" testlist="" _process "$@"; echo "" echo "########################## Preparing to run tests ##########################" preparefailed= if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable" preparefailed=1 fi if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then echo "vtest not found in path, please specify VTEST_PROGRAM environment variable" preparefailed=1 fi if [ $preparefailed ]; then exit 1 fi { read HAPROXY_VERSION; read TARGET; read FEATURES; read SERVICES; } << EOF $($HAPROXY_PROGRAM $HAPROXY_ARGS -vv | grep 'HA-\?Proxy version\|TARGET.*=\|^Feature\|^Available services' | sed 's/.* [:=] //') EOF HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3) echo "Testing with haproxy version: $HAPROXY_VERSION" FEATURES_PATTERN=" $FEATURES " SERVICES_PATTERN=" $SERVICES " TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')" mkdir -p "$TESTDIR" || exit 1 TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1 export TMPDIR="$TESTDIR" export HAPROXY_PROGRAM="$HAPROXY_PROGRAM" if [ -n "$HAPROXY_ARGS" ]; then export HAPROXY_ARGS fi echo "Target : $TARGET" echo "Options : $FEATURES" echo "Services : $SERVICES" echo "########################## Gathering tests to run ##########################" if [ -z "$REGTESTS" ]; then _findtests reg-tests/ else for t in $REGTESTS; do _findtests $t done fi echo "########################## Starting vtest ##########################" echo "Testing with haproxy version: $HAPROXY_VERSION" _vtresult=0 if [ -n "$testlist" ]; then if [ -n "$jobcount" ]; then jobcount="-j $jobcount" fi cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 $keep_logs $verbose $debug $jobcount $vtestparams $testlist" eval $cmd _vtresult=$? else echo "No tests found that meet the required criteria" fi if [ $_vtresult -eq 0 ]; then # all tests were successful, removing tempdir (the last part.) # ignore errors is the directory is not empty or if it does not exist rmdir "$TESTDIR" 2>/dev/null fi if [ -d "${TESTDIR}" ]; then echo "########################## Gathering results ##########################" export TESTDIR find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do if [ ! -e "$i/LOG" ] ; then continue; fi cat <<- EOF | tee -a "$TESTDIR/failedtests.log" $(echo "###### $(cat "$i/INFO") ######") $(echo "## test results in: \"$i\"") $(grep -E -- "^(----|\* diag)" "$i/LOG") EOF done' sh {} + fi exit $_vtresult