qa: Add support for core dumps

Save core dumps when running tests locally
Dump logs to output whenever cores seen

Signed-off-by: David Zafman <dzafman@redhat.com>

qa/run-standalone.sh

@@ -15,9 +15,13 @@ if [ `uname` = FreeBSD ]; then
     # otherwise module prettytable will not be found
     export PYTHONPATH=/usr/local/lib/python2.7/site-packages
     exec_mode=+111
+    KERNCORE="kern.corefile"
+    COREPATTERN="core.%N.%P"
 else
     export PYTHONPATH=/usr/lib/python2.7/dist-packages
     exec_mode=/111
+    KERNCORE="kernel.core_pattern"
+    COREPATTERN="core.%e.%p.%t"
 fi
 
 PATH=$(pwd)/bin:$PATH
@@ -41,6 +45,9 @@ location="../qa/standalone"
 count=0
 errors=0
 userargs=""
+precore="$(sysctl -n $KERNCORE)"
+sudo sysctl -w ${KERNCORE}=${COREPATTERN}
+ulimit -c unlimited
 for f in $(cd $location ; find . -perm $exec_mode -type f)
 do
     f=$(echo $f | sed 's/\.\///')
@@ -82,12 +89,14 @@ do
         if ! PATH=$PATH:bin \
 	    CEPH_ROOT=.. \
 	    CEPH_LIB=lib \
+	    LOCALRUN=yes \
 	    $cmd ; then
           echo "$f .............. FAILED"
           errors=$(expr $errors + 1)
         fi
     fi
 done
+sudo sysctl -w ${KERNCORE}=${precore}
 
 if [ "$errors" != "0" ]; then
     echo "$errors TESTS FAILED, $count TOTAL TESTS"

qa/standalone/ceph-helpers.sh

@@ -33,6 +33,7 @@ fi
 if [ `uname` = FreeBSD ]; then
     SED=gsed
     DIFFCOLOPTS=""
+    KERNCORE="kern.corefile"
 else
     SED=sed
     termwidth=$(stty -a | head -1 | sed -e 's/.*columns \([0-9]*\).*/\1/')
@@ -40,6 +41,7 @@ else
         termwidth="-W ${termwidth}"
     fi
     DIFFCOLOPTS="-y $termwidth"
+    KERNCORE="kernel.core_pattern"
 fi
 
 EXTRA_OPTS=""
@@ -158,11 +160,37 @@ function teardown() {
         && [ $(stat -f -c '%T' .) == "btrfs" ]; then
         __teardown_btrfs $dir
     fi
-    if [ "$dumplogs" = "1" ]; then
+    local cores="no"
+    local pattern="$(sysctl -n $KERNCORE)"
+    # See if we have apport core handling
+    if [ "${pattern:0:1}" = "|" ]; then
+      # TODO: Where can we get the dumps?
+      # Not sure where the dumps really are so this will look in the CWD
+      pattern=""
+    fi
+    # Local we start with core and teuthology ends with core
+    if ls $(dirname $pattern) | grep -q '^core\|core$' ; then
+        cores="yes"
+        if [ -n "$LOCALRUN" ]; then
+	    mkdir /tmp/cores.$$ 2> /dev/null || true
+	    for i in $(ls $(dirname $(sysctl -n $KERNCORE)) | grep '^core\|core$'); do
+		mv $i /tmp/cores.$$
+	    done
+        fi
+    fi
+    if [ "$cores" = "yes" -o "$dumplogs" = "1" ]; then
         display_logs $dir
     fi
     rm -fr $dir
     rm -rf $(get_asok_dir)
+    if [ "$cores" = "yes" ]; then
+        echo "ERROR: Failure due to cores found"
+        if [ -n "$LOCALRUN" ]; then
+	    echo "Find saved core files in /tmp/cores.$$"
+        fi
+        return 1
+    fi
+    return 0
 }
 
 function __teardown_btrfs() {

qa/standalone/special/test-failure.sh

@@ -7,6 +7,11 @@ function run() {
     local dir=$1
     shift
 
+    export CEPH_MON="127.0.0.1:7202" # git grep '\<7202\>' : there must be only one
+    export CEPH_ARGS
+    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
+    CEPH_ARGS+="--mon-host=$CEPH_MON "
+
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for func in $funcs ; do
         setup $dir || return 1
@@ -32,4 +37,12 @@ EOF
     return 1
 }
 
+function TEST_failure_core_only() {
+    local dir=$1
+
+    run_mon $dir a || return 1
+    kill_daemons $dir SEGV mon 5
+    return 0
+}
+
 main test_failure "$@"