REGTEST: replace LEVEL option by a more human readable one.

This patch replaces LEVEL variable by REGTESTS_TYPES variable which is more
mnemonic and human readable. It is uses as a filter to run the reg tests scripts
where a commented #REGTEST_TYPE may be defined to designate their types.
Running the following command:

    $ REGTESTS_TYPES=slow,default

will start all the reg tests where REGTEST_TYPE is defines as 'slow' or 'default'.
Note that 'default' is also the default value of REGTEST_TYPE when not specified
dedicated to run all the current h*.vtc files. When REGTESTS_TYPES is not specified
there is no filter at all. All the tests are run.

This patches also defines REGTEST_TYPE with 'slow' value for all the s*.vtc files,
'bug' value for al the b*.vtc files, 'broken' value for all the k*.vtc files.

Makefile

@@ -166,6 +166,7 @@ ERR =
 
 #### May be used to force running a specific set of reg-tests
 REG_TEST_FILES =
+REG_TEST_SCRIPT=./scripts/run-regtests.sh
 
 #### Compiler-specific flags that may be used to disable some negative over-
 # optimization or to silence some warnings. -fno-strict-aliasing is needed with
@@ -967,17 +968,19 @@ endif
 
 # Target to run the regression testing script files.
 reg-tests:
-	$(Q)./scripts/run-regtests.sh --LEVEL "$(LEVEL)" $(REGTEST_ARGS) $(REG_TEST_FILES)
+	$(Q)$(REG_TEST_SCRIPT) --type "$(REGTESTS_TYPES)" $(REGTEST_ARGS) $(REG_TEST_FILES)
 .PHONY: $(REGTEST_ARGS)
 
 reg-tests-help:
 	@echo
-	@echo "To launch the reg tests for haproxy, first export to your environment VTEST_PROGRAM variable to point to your vtest program:"
+	@echo "To launch the reg tests for haproxy, first export to your environment "
+	@echo "VTEST_PROGRAM variable to point to your vtest program:"
 	@echo "    $$ export VTEST_PROGRAM=/opt/local/bin/vtest"
 	@echo "or"
 	@echo "    $$ setenv VTEST_PROGRAM /opt/local/bin/vtest"
 	@echo
-	@echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM but with ./haproxy as default value."
+	@echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM "
+	@echo "but with ./haproxy as default value."
 	@echo
 	@echo "To run all the tests:"
 	@echo "    $$ make reg-tests"
@@ -985,16 +988,23 @@ reg-tests-help:
 	@echo "You can also set the programs to be used on the command line:"
 	@echo "    $$ VTEST_PROGRAM=<...> HAPROXY_PROGRAM=<...> make reg-tests"
 	@echo
-	@echo "To run tests with specific levels:"
-	@echo "    $$ LEVEL=1,3,4   make reg-tests  #list of levels"
-	@echo "    $$ LEVEL=1-3,5-6 make reg-tests  #list of range of levels"
+	@echo "To run tests with specific types:"
+	@echo "    $$ REGTESTS_TYPES=slow,default make reg-tests"
 	@echo
-	@echo "About the levels:"
-	@echo "    LEVEL 1 scripts are dedicated to pure haproxy compliance tests (prefixed with 'h' letter)."
-	@echo "    LEVEL 2 scripts are slow scripts (prefixed with 's' letter)."
-	@echo "    LEVEL 3 scripts are low interest scripts (prefixed with 'l' letter)."
-	@echo "    LEVEL 4 scripts are in relation with bugs they help to reproduce (prefixed with 'b' letter)."
-	@echo "    LEVEL 5 scripts are scripts triggering known broken behaviors for which there is still no fix (prefixed with 'k' letter)."
-	@echo "    LEVEL 6 scripts are experimental, typically used to develop new scripts (prefixed with 'e' lettre)."
+	@echo "with 'any' as default value for REGTESTS_TYPES variable."
+	@echo
+	@echo "About the reg test types:"
+	@echo "    any         : all the tests without distinction (this is the default"
+	@echo "                  value of REGTESTS_TYPES."
+	@echo "    default     : dedicated to pure haproxy compliance tests."
+	@echo "    slow        : scripts which take non negligible time to run."
+	@echo "    bug         : scripts in relation with bugs they help to reproduce."
+	@echo "    broken      : scripts triggering known broken behaviors for which"
+	@echo "                  there is still no fix."
+	@echo "    experimental: for scripts which are experimental, typically used to"
+	@echo "                  develop new scripts."
+	@echo
+	@echo "Note that 'reg-tests' target run '"$(REG_TEST_SCRIPT)"' script"
+	@echo "(see --help option of this script for more information)."
 
 .PHONY: reg-tests reg-tests-help

reg-tests/checks/s00000.vtc

@@ -3,6 +3,7 @@ feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.8
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
 
 # This script test health-checks for four backends with one server by backend.
 # A syslog server is attached to each backend to check the syslog messages

reg-tests/checks/s00001.vtc

@@ -8,6 +8,7 @@ feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.8
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
 
 # Note that the first syslog message received is: "Proxy <name> started."
 syslog S -repeat 21 -level notice {

reg-tests/checks/s00002.vtc

@@ -3,6 +3,7 @@ feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.8
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
 
 # This script start 40 servers named s0 up to s39.
 # For 0 <= i <= 19:

reg-tests/checks/s00003.vtc

@@ -1,6 +1,8 @@
 varnishtest "Check: smptchk option"
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=slow
+
 barrier b cond 3
 
 syslog S1 -level notice {

reg-tests/compression/s00000.vtc

@@ -3,6 +3,7 @@
 varnishtest "Compression validation"
 #REQUIRE_VERSION=1.6
 #REQUIRE_OPTIONS=ZLIB|SLZ,LUA
+#REGTEST_TYPE=slow
 
 feature ignore_unknown_macro
 

reg-tests/connection/b00000.vtc

@@ -13,6 +13,8 @@ varnishtest "PROXY protocol random failures"
 
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=bug
+
 syslog Slog_1 -repeat 8 -level info {
     recv
     expect ~ "Connect from .* to ${h1_ssl_addr}:${h1_ssl_port}"

reg-tests/http-rules/b00000.vtc

@@ -11,6 +11,8 @@ varnishtest "map_regm get_trash_chunk test"
 feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.6
+#REGTEST_TYPE=bug
+
 syslog S1 -level notice {
     recv
     expect ~ "[^:\\[ ]\\[${h1_pid}\\]: Proxy (fe|be)1 started."

reg-tests/log/b00000.vtc

@@ -18,6 +18,8 @@
 # Note that the problem doesn't happen when using %[src] or %[dst] since
 # all sample expressions set LW_XPRT.
 
+#REGTEST_TYPE=bug
+
 varnishtest "Wrong ip/port logging"
 feature ignore_unknown_macro
 

reg-tests/lua/b00000.vtc

@@ -32,6 +32,7 @@
 
 varnishtest "Basic LUA test h00000"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

reg-tests/lua/b00001.vtc

@@ -21,6 +21,7 @@
 
 varnishtest "LUA bug"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

reg-tests/lua/b00002.vtc

@@ -1,5 +1,6 @@
 varnishtest "Lua: txn:get_priv() scope"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

reg-tests/lua/b00003.vtc

@@ -15,6 +15,7 @@
 
 varnishtest "possible CLOSE-WAIT with '\n' headers"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

reg-tests/mailers/k_healthcheckmail.vtc

@@ -1,6 +1,7 @@
 varnishtest "Lua: txn:get_priv() scope"
 #REQUIRE_OPTIONS=LUA
 #REQUIRE_VERSION=1.6
+#REGTEST_TYPE=broken
 
 feature ignore_unknown_macro
 

reg-tests/peers/s_basic_sync.vtc

@@ -2,6 +2,7 @@ vtest "Basic test for peers protocol"
 feature ignore_unknown_macro
 
 #REQUIRE_VERSION=2.0
+#REGTEST_TYPE=slow
 
 haproxy h1 -arg "-L A" -conf {
     defaults

reg-tests/peers/s_tls_basic_sync.vtc

@@ -3,6 +3,8 @@ vtest "Basic test for peers protocol over SSL/TLS"
 #REQUIRE_VERSION=2.0
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=slow
+
 haproxy h1 -arg "-L A" -conf {
     defaults
         timeout client  1s

reg-tests/seamless-reload/b00000.vtc

@@ -15,6 +15,7 @@ feature ignore_unknown_macro
 #REQUIRE_VERSION=1.8
 # abns@ sockets are not available on freebsd
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=bug
 
 haproxy h1 -W -conf {
   global

reg-tests/server/b00000.vtc

@@ -2,6 +2,8 @@ varnishtest "Set server FQDN via CLI crash"
 
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=bug
+
 # Do nothing. Is there only to create s1_* macros
 server s1 {
 } -start

reg-tests/spoe/b00000.vtc

@@ -5,6 +5,8 @@
 # segmentaion fault when first errors are catched, when the "filter spoe" line is
 # parsed.
 
+#REGTEST_TYPE=bug
+
 varnishtest "SPOE bug: missing configuration file"
 
 feature ignore_unknown_macro

reg-tests/ssl/b00000.vtc

@@ -15,6 +15,7 @@
 #    #define SSL_set_app_data(s,arg)     (SSL_set_ex_data(s,0,(char *)arg))
 #    #define SSL_get_app_data(s)      (SSL_get_ex_data(s,0))
 
+#REGTEST_TYPE=bug
 
 varnishtest "OpenSSL bug: Random crashes"
 #REQUIRE_OPTIONS=OPENSSL

reg-tests/stick-table/b00000.vtc

@@ -1,5 +1,7 @@
 # Shipped with the commit fixing the bug.
 
+#REGTEST_TYPE=bug
+
 varnishtest "Stick Table: Crash when accessing unknown key."
 feature ignore_unknown_macro
 

reg-tests/stick-table/b00001.vtc

@@ -14,6 +14,8 @@
 #
 # This should be backported to 1.8
 
+#REGTEST_TYPE=bug
+
 varnishtest "stick-tables: Test expirations when used with table_*"
 
 # As some macros for haproxy are used in this file, this line is mandatory.

scripts/run-regtests.sh

@@ -28,6 +28,13 @@ _help()
     --vtestparams <ARGS>, passes custom ARGS to vtest
       run-regtests.sh --vtestparams "-n 10"
 
+    --type <reg tests types> filter the types of the tests to be run, depending on
+      the commented REGTESTS_TYPE variable value in each VTC file.
+      The value of REGTESTS_TYPE supported are: default, slow, bug, broken and
+      experimental. When not specified, it is set to 'default' as default value.
+
+      run-regtest.sh --type slow,default
+
     --clean to cleanup previous reg-tests log directories and exit
       run-regtests.sh --clean
 
@@ -105,39 +112,6 @@ add_range_to_test_list()
     echo $list
 }
 
-
-build_test_list()
-{
-    # Remove any spacing character
-    LEVEL="$(echo $LEVEL | tr -d ' ')"
-    # Replave any comma character by a space character
-    LEVEL="$(echo $LEVEL | tr ',' ' ')"
-    list=
-    for range in $LEVEL ; do
-        if [ -z "$list" ] ; then
-            list=$(add_range_to_test_list $range)
-        else
-            list="$list $(add_range_to_test_list $range)"
-        fi
-    done
-
-    echo $list
-}
-
-build_find_expr()
-{
-    expr=
-    for i in $@; do
-        if [ -z "$expr" ] ; then
-            expr="-name \"$i\""
-        else
-            expr="$expr -o -name \"$i\""
-        fi
-    done
-
-    echo $expr
-}
-
 _startswith() {
   _str="$1"
   _sub="$2"
@@ -146,20 +120,25 @@ _startswith() {
 
 _findtests() {
   set -f
-  LEVEL=${LEVEL:-0};
-  list=$(build_test_list "$LEVEL")
-  if [ -z "$list" ] ; then
-      echo "Invalid level specification '"$LEVEL"' or no file was found."
-      exit 1
-  fi
-  EXPR=$(build_find_expr $list)
 
-  for i in $( find "$1" $(eval echo $EXPR) ); do
+  REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
+  any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
+  for i in $( find "$1" -name *.vtc ); do
     skiptest=
     require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
     require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
     require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed  -e 's/,/ /g')"
     exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed  -e 's/,/ /g')"
+    if [ $any_test -ne 1 ] ; then
+        regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
+        if [ -z $regtest_type ] ; then
+            regtest_type=default
+        fi
+        if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
+            echo "  Skip $i because its type '"$regtest_type"' is excluded"
+            skiptest=1
+        fi
+    fi
 
     requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed  -e 's/,.*//')"
     if [ -n "$requiredoption" ]; then
@@ -264,8 +243,8 @@ _process() {
         --keep-logs)
           keep_logs="-L"
           ;;
-        --LEVEL)
-          LEVEL="$2"
+        --type)
+	      REGTESTS_TYPES="$2"
 	      shift
 	      ;;
         --use-htx)