From 70b1303a08ce30ca8650a231505a9c8a819be1e0 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Thu, 31 Mar 2016 11:42:13 +0300 Subject: [PATCH 1/3] qa/workunits/rbd: improvements for manual testing Signed-off-by: Mykola Golub --- qa/workunits/rbd/rbd_mirror.sh | 124 ++++++++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 17 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 872f44efbca..7161e2fa3c1 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -40,13 +40,24 @@ # ceph --cluster local -s # rbd --cluster remote -p mirror ls # rbd --cluster remote -p mirror journal status --image test -# ceph --admin-daemon rbd-mirror.asok help +# ceph --admin-daemon rbd-mirror.local.local.asok help +# ... +# +# Also you can execute commands (functions) from the script: +# +# cd $CEPH_SRC_PATH +# export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror +# ../qa/workunits/rbd/rbd_mirror.sh status +# ../qa/workunits/rbd/rbd_mirror.sh stop_mirror local +# ../qa/workunits/rbd/rbd_mirror.sh start_mirror remote remote +# ../qa/workunits/rbd/rbd_mirror.sh flush remote # ... # # Eventually, run the cleanup: # # cd $CEPH_SRC_PATH -# ../qa/workunits/rbd/rbd_mirror.sh cleanup /tmp/tmp.rbd_mirror +# RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ +# ../qa/workunits/rbd/rbd_mirror.sh cleanup # LOC_DAEMON=local @@ -163,16 +174,96 @@ stop_mirror() rm -f $(daemon_pid_file "${daemon}") } +status() +{ + local cluster d daemon image + + for cluster in ${LOC_CLUSTER} ${RMT_CLUSTER} + do + echo "${cluster} status" + ceph --cluster ${cluster} -s + echo + + echo "${cluster} ${POOL} images" + rbd --cluster ${cluster} -p ${POOL} ls + echo + + for image in `rbd --cluster ${cluster} -p ${POOL} ls 2>/dev/null` + do + echo "image ${image} journal status" + rbd --cluster ${cluster} -p ${POOL} journal status --image ${image} + echo + done + done + + local ret + + for d in "${LOC_DAEMON}@${LOC_CLUSTER}" "${RMT_DAEMON}@${RMT_CLUSTER}" + do + daemon=${d%%@*} + cluster=${d#*@} + + local pid_file=$(daemon_pid_file ${daemon}) + if [ ! -e ${pid_file} ] + then + echo "${daemon} rbd-mirror not running or unknown" \ + "(${pid_file} not exist)" + continue + fi + + local pid + pid=$(cat ${pid_file} 2>/dev/null) || : + if [ -z "${pid}" ] + then + echo "${daemon} rbd-mirror not running or unknown" \ + "(can't find pid using ${pid_file})" + ret=1 + continue + fi + + echo "${daemon} rbd-mirror process in ps output:" + if ps auxww | + awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}' + then + echo + echo "${daemon} rbd-mirror not running" \ + "(can't find pid $pid in ps output)" + ret=1 + continue + fi + echo + + local asok_file=$(daemon_asok_file ${daemon} ${cluster}) + if [ ! -S "${asok_file}" ] + then + echo "${daemon} rbd-mirror asok is unknown (${asok_file} not exits)" + ret=1 + continue + fi + + echo "${daemon} rbd-mirror status" + ceph --admin-daemon ${asok_file} rbd mirror status + echo + done + + return ${ret} +} + flush() { local daemon=$1 local image=$2 + local cmd="rbd mirror flush" + + if [ -n "${image}" ] + then + cmd="${cmd} ${POOL}/${image}" + fi local asok_file=$(daemon_asok_file "${daemon}" "${daemon}") - test -n "${asok_file}" + test -S "${asok_file}" - ceph --admin-daemon ${asok_file} \ - rbd mirror flush ${POOL}/${image} + ceph --admin-daemon ${asok_file} ${cmd} } test_image_replay_state() @@ -184,7 +275,7 @@ test_image_replay_state() local current_state=stopped local asok_file=$(daemon_asok_file "${daemon}" "${cluster}") - test -n "${asok_file}" + test -S "${asok_file}" ceph --admin-daemon ${asok_file} help | fgrep "\"rbd mirror status ${POOL}/${image}\"" && current_state=started @@ -341,19 +432,18 @@ promote_image() # Main # -if [ "$1" = clean ]; then - TEMPDIR=$2 - - if [ -z "${TEMPDIR}" -a -n "${RBD_MIRROR_TEMDIR}" ]; then - TEMPDIR="${RBD_MIRROR_TEMDIR}" +if [ "$#" -gt 0 ] +then + if [ -z "${RBD_MIRROR_TEMDIR}" ] + then + echo "RBD_MIRROR_TEMDIR is not set" >&2 + exit 1 fi - test -n "${TEMPDIR}" - - RBD_MIRROR_NOCLEANUP= - - cleanup - exit + TEMPDIR="${RBD_MIRROR_TEMDIR}" + cd ${TEMPDIR} + $@ + exit $? fi set -xe From 22260190371768e92b8032377a8345641951e769 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Thu, 31 Mar 2016 13:34:30 +0300 Subject: [PATCH 2/3] qa/workunits/rbd: add helper to determine rbd-mirror local cluster Signed-off-by: Mykola Golub --- qa/workunits/rbd/rbd_mirror.sh | 60 ++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 7161e2fa3c1..528c1f66421 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -174,9 +174,22 @@ stop_mirror() rm -f $(daemon_pid_file "${daemon}") } +daemon_local_cluster() +{ + local daemon=$1 + local pid_file=$(daemon_pid_file $daemon) + + pid=$(cat $(daemon_pid_file "${daemon}")) + + test -n "${pid}" + + ps auxww | awk -v pid=${pid} '$2 == pid' | + sed -nEe 's/^.*--cluster +([^ ]+).*$/\1/p' +} + status() { - local cluster d daemon image + local cluster daemon image for cluster in ${LOC_CLUSTER} ${RMT_CLUSTER} do @@ -198,11 +211,8 @@ status() local ret - for d in "${LOC_DAEMON}@${LOC_CLUSTER}" "${RMT_DAEMON}@${RMT_CLUSTER}" + for daemon in "${LOC_DAEMON}" "${RMT_DAEMON}" do - daemon=${d%%@*} - cluster=${d#*@} - local pid_file=$(daemon_pid_file ${daemon}) if [ ! -e ${pid_file} ] then @@ -233,6 +243,8 @@ status() fi echo + cluster=$(daemon_local_cluster ${daemon}) + local asok_file=$(daemon_asok_file ${daemon} ${cluster}) if [ ! -S "${asok_file}" ] then @@ -260,7 +272,8 @@ flush() cmd="${cmd} ${POOL}/${image}" fi - local asok_file=$(daemon_asok_file "${daemon}" "${daemon}") + local cluster=$(daemon_local_cluster "${daemon}") + local asok_file=$(daemon_asok_file "${daemon}" "${cluster}") test -S "${asok_file}" ceph --admin-daemon ${asok_file} ${cmd} @@ -269,11 +282,11 @@ flush() test_image_replay_state() { local daemon=$1 - local cluster=$2 - local image=$3 - local test_state=$4 + local image=$2 + local test_state=$3 local current_state=stopped + local cluster=$(daemon_local_cluster "${daemon}") local asok_file=$(daemon_asok_file "${daemon}" "${cluster}") test -S "${asok_file}" @@ -285,15 +298,14 @@ test_image_replay_state() wait_for_image_replay_state() { local daemon=$1 - local cluster=$2 - local image=$3 - local state=$4 + local image=$2 + local state=$3 local s # TODO: add a way to force rbd-mirror to update replayers for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do sleep ${s} - test_image_replay_state "${daemon}" "${cluster}" "${image}" "${state}" && return 0 + test_image_replay_state "${daemon}" "${image}" "${state}" && return 0 done return 1 } @@ -301,19 +313,17 @@ wait_for_image_replay_state() wait_for_image_replay_started() { local daemon=$1 - local cluster=$2 - local image=$3 + local image=$2 - wait_for_image_replay_state "${daemon}" "${cluster}" "${image}" started + wait_for_image_replay_state "${daemon}" "${image}" started } wait_for_image_replay_stopped() { local daemon=$1 - local cluster=$2 - local image=$3 + local image=$2 - wait_for_image_replay_state "${daemon}" "${cluster}" "${image}" stopped + wait_for_image_replay_state "${daemon}" "${image}" stopped } get_position() @@ -453,7 +463,7 @@ setup echo "TEST: add image and test replay" image=test create_remote_image ${image} -wait_for_image_replay_started ${LOC_DAEMON} ${LOC_CLUSTER} ${image} +wait_for_image_replay_started ${LOC_DAEMON} ${image} write_image ${RMT_CLUSTER} ${image} 100 wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image} compare_images ${image} @@ -464,7 +474,7 @@ image1=test1 create_remote_image ${image1} write_image ${RMT_CLUSTER} ${image1} 100 start_mirror ${LOC_DAEMON} ${LOC_CLUSTER} -wait_for_image_replay_started ${LOC_DAEMON} ${LOC_CLUSTER} ${image1} +wait_for_image_replay_started ${LOC_DAEMON} ${image1} wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image1} compare_images ${image1} @@ -478,18 +488,18 @@ start_mirror ${RMT_DAEMON} ${RMT_CLUSTER} # failover demote_image ${RMT_CLUSTER} ${image} -wait_for_image_replay_stopped ${LOC_DAEMON} ${LOC_CLUSTER} ${image} +wait_for_image_replay_stopped ${LOC_DAEMON} ${image} promote_image ${LOC_CLUSTER} ${image} -wait_for_image_replay_started ${RMT_DAEMON} ${RMT_CLUSTER} ${image} +wait_for_image_replay_started ${RMT_DAEMON} ${image} write_image ${LOC_CLUSTER} ${image} 100 wait_for_replay_complete ${RMT_DAEMON} ${LOC_CLUSTER} ${image} compare_images ${image} # failback demote_image ${LOC_CLUSTER} ${image} -wait_for_image_replay_stopped ${RMT_DAEMON} ${RMT_CLUSTER} ${image} +wait_for_image_replay_stopped ${RMT_DAEMON} ${image} promote_image ${RMT_CLUSTER} ${image} -wait_for_image_replay_started ${LOC_DAEMON} ${LOC_CLUSTER} ${image} +wait_for_image_replay_started ${LOC_DAEMON} ${image} write_image ${RMT_CLUSTER} ${image} 100 wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image} compare_images ${image} From 4e1e81a2a720d7fa83bdc2e0c8d4befbb134cb49 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Thu, 31 Mar 2016 13:57:17 +0300 Subject: [PATCH 3/3] qa/workunits/rbd: use cluster1/2 instead local/remote as cluster names After adding tests that start daemons in both clusters, local/remote names look confusing. Signed-off-by: Mykola Golub --- qa/workunits/rbd/rbd_mirror.sh | 143 +++++++++++++++------------------ 1 file changed, 64 insertions(+), 79 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror.sh b/qa/workunits/rbd/rbd_mirror.sh index 528c1f66421..4d3c806a7c8 100755 --- a/qa/workunits/rbd/rbd_mirror.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -35,12 +35,12 @@ # # cd /tmp/tmp.rbd_mirror # ls -# less rbd-mirror.local.local..log -# ceph --cluster remote -s -# ceph --cluster local -s -# rbd --cluster remote -p mirror ls -# rbd --cluster remote -p mirror journal status --image test -# ceph --admin-daemon rbd-mirror.local.local.asok help +# less rbd-mirror.cluster1_daemon.cluster1.log +# ceph --cluster cluster1 -s +# ceph --cluster cluster1 -s +# rbd --cluster cluster2 -p mirror ls +# rbd --cluster cluster2 -p mirror journal status --image test +# ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.asok help # ... # # Also you can execute commands (functions) from the script: @@ -48,9 +48,9 @@ # cd $CEPH_SRC_PATH # export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror # ../qa/workunits/rbd/rbd_mirror.sh status -# ../qa/workunits/rbd/rbd_mirror.sh stop_mirror local -# ../qa/workunits/rbd/rbd_mirror.sh start_mirror remote remote -# ../qa/workunits/rbd/rbd_mirror.sh flush remote +# ../qa/workunits/rbd/rbd_mirror.sh stop_mirror cluster1_daemon +# ../qa/workunits/rbd/rbd_mirror.sh start_mirror cluster2_daemon cluster2 +# ../qa/workunits/rbd/rbd_mirror.sh flush cluster2_daemon # ... # # Eventually, run the cleanup: @@ -60,10 +60,10 @@ # ../qa/workunits/rbd/rbd_mirror.sh cleanup # -LOC_DAEMON=local -RMT_DAEMON=remote -LOC_CLUSTER=local -RMT_CLUSTER=remote +CLUSTER1=cluster1 +CLUSTER2=cluster2 +CLUSTER1_DAEMON=${CLUSTER1}_daemon +CLUSTER2_DAEMON=${CLUSTER2}_daemon POOL=mirror SRC_DIR=$(readlink -f $(dirname $0)/../../../src) TEMPDIR= @@ -100,26 +100,24 @@ setup() fi cd ${SRC_DIR} - ./mstart.sh ${LOC_CLUSTER} -n - ./mstart.sh ${RMT_CLUSTER} -n + ./mstart.sh ${CLUSTER1} -n + ./mstart.sh ${CLUSTER2} -n - ln -s $(readlink -f run/${LOC_CLUSTER}/ceph.conf) \ - ${TEMPDIR}/${LOC_CLUSTER}.conf - ln -s $(readlink -f run/${RMT_CLUSTER}/ceph.conf) \ - ${TEMPDIR}/${RMT_CLUSTER}.conf + ln -s $(readlink -f run/${CLUSTER1}/ceph.conf) \ + ${TEMPDIR}/${CLUSTER1}.conf + ln -s $(readlink -f run/${CLUSTER2}/ceph.conf) \ + ${TEMPDIR}/${CLUSTER2}.conf cd ${TEMPDIR} - start_mirror ${LOC_DAEMON} ${LOC_CLUSTER} + ceph --cluster ${CLUSTER1} osd pool create ${POOL} 64 64 + ceph --cluster ${CLUSTER2} osd pool create ${POOL} 64 64 - ceph --cluster ${LOC_CLUSTER} osd pool create ${POOL} 64 64 - ceph --cluster ${RMT_CLUSTER} osd pool create ${POOL} 64 64 + rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} pool + rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} pool - rbd --cluster ${LOC_CLUSTER} mirror pool enable ${POOL} pool - rbd --cluster ${RMT_CLUSTER} mirror pool enable ${POOL} pool - - rbd --cluster ${LOC_CLUSTER} mirror pool peer add ${POOL} ${RMT_CLUSTER} - rbd --cluster ${RMT_CLUSTER} mirror pool peer add ${POOL} ${LOC_CLUSTER} + rbd --cluster ${CLUSTER1} mirror pool peer add ${POOL} ${CLUSTER2} + rbd --cluster ${CLUSTER2} mirror pool peer add ${POOL} ${CLUSTER1} } cleanup() @@ -128,13 +126,13 @@ cleanup() set +e - stop_mirror ${LOC_DAEMON} - stop_mirror ${RMT_DAEMON} + stop_mirror ${CLUSTER1_DAEMON} + stop_mirror ${CLUSTER2_DAEMON} cd ${SRC_DIR} - ./mstop.sh ${LOC_CLUSTER} - ./mstop.sh ${RMT_CLUSTER} + ./mstop.sh ${CLUSTER1} + ./mstop.sh ${CLUSTER2} rm -Rf ${TEMPDIR} } @@ -169,8 +167,8 @@ stop_mirror() done ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}' fi - rm -f $(daemon_asok_file "${daemon}" "${LOC_CLUSTER}") - rm -f $(daemon_asok_file "${daemon}" "${RMT_CLUSTER}") + rm -f $(daemon_asok_file "${daemon}" "${CLUSTER1}") + rm -f $(daemon_asok_file "${daemon}" "${CLUSTER2}") rm -f $(daemon_pid_file "${daemon}") } @@ -191,7 +189,7 @@ status() { local cluster daemon image - for cluster in ${LOC_CLUSTER} ${RMT_CLUSTER} + for cluster in ${CLUSTER1} ${CLUSTER2} do echo "${cluster} status" ceph --cluster ${cluster} -s @@ -211,7 +209,7 @@ status() local ret - for daemon in "${LOC_DAEMON}" "${RMT_DAEMON}" + for daemon in "${CLUSTER1_DAEMON}" "${CLUSTER2_DAEMON}" do local pid_file=$(daemon_pid_file ${daemon}) if [ ! -e ${pid_file} ] @@ -335,7 +333,7 @@ get_position() # Parse line like below, looking for the first position # [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]] - local status_log=${TEMPDIR}/${RMT_CLUSTER}-${POOL}-${image}.status + local status_log=${TEMPDIR}/${CLUSTER2}-${POOL}-${image}.status rbd --cluster ${cluster} -p ${POOL} journal status --image ${image} | tee ${status_log} >&2 sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*$/\1/p' \ @@ -384,20 +382,6 @@ create_image() --image-feature exclusive-lock --image-feature journaling ${image} } -create_remote_image() -{ - local image=$1 - - create_image ${RMT_CLUSTER} ${image} -} - -create_local_image() -{ - local image=$1 - - create_image ${LOC_CLUSTER} ${image} -} - write_image() { local cluster=$1 @@ -413,12 +397,12 @@ compare_images() { local image=$1 - local rmt_export=${TEMPDIR}/${RMT_CLUSTER}-${POOL}-${image}.export - local loc_export=${TEMPDIR}/${LOC_CLUSTER}-${POOL}-${image}.export + local rmt_export=${TEMPDIR}/${CLUSTER2}-${POOL}-${image}.export + local loc_export=${TEMPDIR}/${CLUSTER1}-${POOL}-${image}.export rm -f ${rmt_export} ${loc_export} - rbd --cluster ${RMT_CLUSTER} -p ${POOL} export ${image} ${rmt_export} - rbd --cluster ${LOC_CLUSTER} -p ${POOL} export ${image} ${loc_export} + rbd --cluster ${CLUSTER2} -p ${POOL} export ${image} ${rmt_export} + rbd --cluster ${CLUSTER1} -p ${POOL} export ${image} ${loc_export} cmp ${rmt_export} ${loc_export} } @@ -461,47 +445,48 @@ set -xe setup echo "TEST: add image and test replay" +start_mirror ${CLUSTER1_DAEMON} ${CLUSTER1} image=test -create_remote_image ${image} -wait_for_image_replay_started ${LOC_DAEMON} ${image} -write_image ${RMT_CLUSTER} ${image} 100 -wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image} +create_image ${CLUSTER2} ${image} +wait_for_image_replay_started ${CLUSTER1_DAEMON} ${image} +write_image ${CLUSTER2} ${image} 100 +wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image} compare_images ${image} echo "TEST: stop mirror, add image, start mirror and test replay" -stop_mirror ${LOC_DAEMON} +stop_mirror ${CLUSTER1_DAEMON} image1=test1 -create_remote_image ${image1} -write_image ${RMT_CLUSTER} ${image1} 100 -start_mirror ${LOC_DAEMON} ${LOC_CLUSTER} -wait_for_image_replay_started ${LOC_DAEMON} ${image1} -wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image1} +create_image ${CLUSTER2} ${image1} +write_image ${CLUSTER2} ${image1} 100 +start_mirror ${CLUSTER1_DAEMON} ${CLUSTER1} +wait_for_image_replay_started ${CLUSTER1_DAEMON} ${image1} +wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image1} compare_images ${image1} echo "TEST: test the first image is replaying after restart" -write_image ${RMT_CLUSTER} ${image} 100 -wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image} +write_image ${CLUSTER2} ${image} 100 +wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image} compare_images ${image} echo "TEST: failover and failback" -start_mirror ${RMT_DAEMON} ${RMT_CLUSTER} +start_mirror ${CLUSTER2_DAEMON} ${CLUSTER2} # failover -demote_image ${RMT_CLUSTER} ${image} -wait_for_image_replay_stopped ${LOC_DAEMON} ${image} -promote_image ${LOC_CLUSTER} ${image} -wait_for_image_replay_started ${RMT_DAEMON} ${image} -write_image ${LOC_CLUSTER} ${image} 100 -wait_for_replay_complete ${RMT_DAEMON} ${LOC_CLUSTER} ${image} +demote_image ${CLUSTER2} ${image} +wait_for_image_replay_stopped ${CLUSTER1_DAEMON} ${image} +promote_image ${CLUSTER1} ${image} +wait_for_image_replay_started ${CLUSTER2_DAEMON} ${image} +write_image ${CLUSTER1} ${image} 100 +wait_for_replay_complete ${CLUSTER2_DAEMON} ${CLUSTER1} ${image} compare_images ${image} # failback -demote_image ${LOC_CLUSTER} ${image} -wait_for_image_replay_stopped ${RMT_DAEMON} ${image} -promote_image ${RMT_CLUSTER} ${image} -wait_for_image_replay_started ${LOC_DAEMON} ${image} -write_image ${RMT_CLUSTER} ${image} 100 -wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image} +demote_image ${CLUSTER1} ${image} +wait_for_image_replay_stopped ${CLUSTER2_DAEMON} ${image} +promote_image ${CLUSTER2} ${image} +wait_for_image_replay_started ${CLUSTER1_DAEMON} ${image} +write_image ${CLUSTER2} ${image} 100 +wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image} compare_images ${image} echo OK