mirror of
https://github.com/ceph/ceph
synced 2024-12-19 01:46:00 +00:00
Merge pull request #8325 from trociny/wip-rbd-mirror-test-misc
test: rbd-mirror: script improvements for manual testing Reviewed-by: Josh Durgin <jdurgin@redhat.com>
This commit is contained in:
Josh Durgin
commit
c9ca7692e4
@ -35,24 +35,35 @@
|
||||
#
|
||||
# cd /tmp/tmp.rbd_mirror
|
||||
# ls
|
||||
# less rbd-mirror.local.local.<pid>.log
|
||||
# ceph --cluster remote -s
|
||||
# ceph --cluster local -s
|
||||
# rbd --cluster remote -p mirror ls
|
||||
# rbd --cluster remote -p mirror journal status --image test
|
||||
# ceph --admin-daemon rbd-mirror.asok help
|
||||
# less rbd-mirror.cluster1_daemon.cluster1.log
|
||||
# ceph --cluster cluster1 -s
|
||||
# ceph --cluster cluster1 -s
|
||||
# rbd --cluster cluster2 -p mirror ls
|
||||
# rbd --cluster cluster2 -p mirror journal status --image test
|
||||
# ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.asok help
|
||||
# ...
|
||||
#
|
||||
# Also you can execute commands (functions) from the script:
|
||||
#
|
||||
# cd $CEPH_SRC_PATH
|
||||
# export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror
|
||||
# ../qa/workunits/rbd/rbd_mirror.sh status
|
||||
# ../qa/workunits/rbd/rbd_mirror.sh stop_mirror cluster1_daemon
|
||||
# ../qa/workunits/rbd/rbd_mirror.sh start_mirror cluster2_daemon cluster2
|
||||
# ../qa/workunits/rbd/rbd_mirror.sh flush cluster2_daemon
|
||||
# ...
|
||||
#
|
||||
# Eventually, run the cleanup:
|
||||
#
|
||||
# cd $CEPH_SRC_PATH
|
||||
# ../qa/workunits/rbd/rbd_mirror.sh cleanup /tmp/tmp.rbd_mirror
|
||||
# RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \
|
||||
# ../qa/workunits/rbd/rbd_mirror.sh cleanup
|
||||
#
|
||||
|
||||
LOC_DAEMON=local
|
||||
RMT_DAEMON=remote
|
||||
LOC_CLUSTER=local
|
||||
RMT_CLUSTER=remote
|
||||
CLUSTER1=cluster1
|
||||
CLUSTER2=cluster2
|
||||
CLUSTER1_DAEMON=${CLUSTER1}_daemon
|
||||
CLUSTER2_DAEMON=${CLUSTER2}_daemon
|
||||
POOL=mirror
|
||||
SRC_DIR=$(readlink -f $(dirname $0)/../../../src)
|
||||
TEMPDIR=
|
||||
@ -89,26 +100,24 @@ setup()
|
||||
fi
|
||||
|
||||
cd ${SRC_DIR}
|
||||
./mstart.sh ${LOC_CLUSTER} -n
|
||||
./mstart.sh ${RMT_CLUSTER} -n
|
||||
./mstart.sh ${CLUSTER1} -n
|
||||
./mstart.sh ${CLUSTER2} -n
|
||||
|
||||
ln -s $(readlink -f run/${LOC_CLUSTER}/ceph.conf) \
|
||||
${TEMPDIR}/${LOC_CLUSTER}.conf
|
||||
ln -s $(readlink -f run/${RMT_CLUSTER}/ceph.conf) \
|
||||
${TEMPDIR}/${RMT_CLUSTER}.conf
|
||||
ln -s $(readlink -f run/${CLUSTER1}/ceph.conf) \
|
||||
${TEMPDIR}/${CLUSTER1}.conf
|
||||
ln -s $(readlink -f run/${CLUSTER2}/ceph.conf) \
|
||||
${TEMPDIR}/${CLUSTER2}.conf
|
||||
|
||||
cd ${TEMPDIR}
|
||||
|
||||
start_mirror ${LOC_DAEMON} ${LOC_CLUSTER}
|
||||
ceph --cluster ${CLUSTER1} osd pool create ${POOL} 64 64
|
||||
ceph --cluster ${CLUSTER2} osd pool create ${POOL} 64 64
|
||||
|
||||
ceph --cluster ${LOC_CLUSTER} osd pool create ${POOL} 64 64
|
||||
ceph --cluster ${RMT_CLUSTER} osd pool create ${POOL} 64 64
|
||||
rbd --cluster ${CLUSTER1} mirror pool enable ${POOL} pool
|
||||
rbd --cluster ${CLUSTER2} mirror pool enable ${POOL} pool
|
||||
|
||||
rbd --cluster ${LOC_CLUSTER} mirror pool enable ${POOL} pool
|
||||
rbd --cluster ${RMT_CLUSTER} mirror pool enable ${POOL} pool
|
||||
|
||||
rbd --cluster ${LOC_CLUSTER} mirror pool peer add ${POOL} ${RMT_CLUSTER}
|
||||
rbd --cluster ${RMT_CLUSTER} mirror pool peer add ${POOL} ${LOC_CLUSTER}
|
||||
rbd --cluster ${CLUSTER1} mirror pool peer add ${POOL} ${CLUSTER2}
|
||||
rbd --cluster ${CLUSTER2} mirror pool peer add ${POOL} ${CLUSTER1}
|
||||
}
|
||||
|
||||
cleanup()
|
||||
@ -117,13 +126,13 @@ cleanup()
|
||||
|
||||
set +e
|
||||
|
||||
stop_mirror ${LOC_DAEMON}
|
||||
stop_mirror ${RMT_DAEMON}
|
||||
stop_mirror ${CLUSTER1_DAEMON}
|
||||
stop_mirror ${CLUSTER2_DAEMON}
|
||||
|
||||
cd ${SRC_DIR}
|
||||
|
||||
./mstop.sh ${LOC_CLUSTER}
|
||||
./mstop.sh ${RMT_CLUSTER}
|
||||
./mstop.sh ${CLUSTER1}
|
||||
./mstop.sh ${CLUSTER2}
|
||||
|
||||
rm -Rf ${TEMPDIR}
|
||||
}
|
||||
@ -158,33 +167,126 @@ stop_mirror()
|
||||
done
|
||||
ps auxww | awk -v pid=${pid} '$2 == pid {print; exit 1}'
|
||||
fi
|
||||
rm -f $(daemon_asok_file "${daemon}" "${LOC_CLUSTER}")
|
||||
rm -f $(daemon_asok_file "${daemon}" "${RMT_CLUSTER}")
|
||||
rm -f $(daemon_asok_file "${daemon}" "${CLUSTER1}")
|
||||
rm -f $(daemon_asok_file "${daemon}" "${CLUSTER2}")
|
||||
rm -f $(daemon_pid_file "${daemon}")
|
||||
}
|
||||
|
||||
daemon_local_cluster()
|
||||
{
|
||||
local daemon=$1
|
||||
local pid_file=$(daemon_pid_file $daemon)
|
||||
|
||||
pid=$(cat $(daemon_pid_file "${daemon}"))
|
||||
|
||||
test -n "${pid}"
|
||||
|
||||
ps auxww | awk -v pid=${pid} '$2 == pid' |
|
||||
sed -nEe 's/^.*--cluster +([^ ]+).*$/\1/p'
|
||||
}
|
||||
|
||||
status()
|
||||
{
|
||||
local cluster daemon image
|
||||
|
||||
for cluster in ${CLUSTER1} ${CLUSTER2}
|
||||
do
|
||||
echo "${cluster} status"
|
||||
ceph --cluster ${cluster} -s
|
||||
echo
|
||||
|
||||
echo "${cluster} ${POOL} images"
|
||||
rbd --cluster ${cluster} -p ${POOL} ls
|
||||
echo
|
||||
|
||||
for image in `rbd --cluster ${cluster} -p ${POOL} ls 2>/dev/null`
|
||||
do
|
||||
echo "image ${image} journal status"
|
||||
rbd --cluster ${cluster} -p ${POOL} journal status --image ${image}
|
||||
echo
|
||||
done
|
||||
done
|
||||
|
||||
local ret
|
||||
|
||||
for daemon in "${CLUSTER1_DAEMON}" "${CLUSTER2_DAEMON}"
|
||||
do
|
||||
local pid_file=$(daemon_pid_file ${daemon})
|
||||
if [ ! -e ${pid_file} ]
|
||||
then
|
||||
echo "${daemon} rbd-mirror not running or unknown" \
|
||||
"(${pid_file} not exist)"
|
||||
continue
|
||||
fi
|
||||
|
||||
local pid
|
||||
pid=$(cat ${pid_file} 2>/dev/null) || :
|
||||
if [ -z "${pid}" ]
|
||||
then
|
||||
echo "${daemon} rbd-mirror not running or unknown" \
|
||||
"(can't find pid using ${pid_file})"
|
||||
ret=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "${daemon} rbd-mirror process in ps output:"
|
||||
if ps auxww |
|
||||
awk -v pid=${pid} 'NR == 1 {print} $2 == pid {print; exit 1}'
|
||||
then
|
||||
echo
|
||||
echo "${daemon} rbd-mirror not running" \
|
||||
"(can't find pid $pid in ps output)"
|
||||
ret=1
|
||||
continue
|
||||
fi
|
||||
echo
|
||||
|
||||
cluster=$(daemon_local_cluster ${daemon})
|
||||
|
||||
local asok_file=$(daemon_asok_file ${daemon} ${cluster})
|
||||
if [ ! -S "${asok_file}" ]
|
||||
then
|
||||
echo "${daemon} rbd-mirror asok is unknown (${asok_file} not exits)"
|
||||
ret=1
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "${daemon} rbd-mirror status"
|
||||
ceph --admin-daemon ${asok_file} rbd mirror status
|
||||
echo
|
||||
done
|
||||
|
||||
return ${ret}
|
||||
}
|
||||
|
||||
flush()
|
||||
{
|
||||
local daemon=$1
|
||||
local image=$2
|
||||
local cmd="rbd mirror flush"
|
||||
|
||||
local asok_file=$(daemon_asok_file "${daemon}" "${daemon}")
|
||||
test -n "${asok_file}"
|
||||
if [ -n "${image}" ]
|
||||
then
|
||||
cmd="${cmd} ${POOL}/${image}"
|
||||
fi
|
||||
|
||||
ceph --admin-daemon ${asok_file} \
|
||||
rbd mirror flush ${POOL}/${image}
|
||||
local cluster=$(daemon_local_cluster "${daemon}")
|
||||
local asok_file=$(daemon_asok_file "${daemon}" "${cluster}")
|
||||
test -S "${asok_file}"
|
||||
|
||||
ceph --admin-daemon ${asok_file} ${cmd}
|
||||
}
|
||||
|
||||
test_image_replay_state()
|
||||
{
|
||||
local daemon=$1
|
||||
local cluster=$2
|
||||
local image=$3
|
||||
local test_state=$4
|
||||
local image=$2
|
||||
local test_state=$3
|
||||
local current_state=stopped
|
||||
|
||||
local cluster=$(daemon_local_cluster "${daemon}")
|
||||
local asok_file=$(daemon_asok_file "${daemon}" "${cluster}")
|
||||
test -n "${asok_file}"
|
||||
test -S "${asok_file}"
|
||||
|
||||
ceph --admin-daemon ${asok_file} help |
|
||||
fgrep "\"rbd mirror status ${POOL}/${image}\"" && current_state=started
|
||||
@ -194,15 +296,14 @@ test_image_replay_state()
|
||||
wait_for_image_replay_state()
|
||||
{
|
||||
local daemon=$1
|
||||
local cluster=$2
|
||||
local image=$3
|
||||
local state=$4
|
||||
local image=$2
|
||||
local state=$3
|
||||
local s
|
||||
|
||||
# TODO: add a way to force rbd-mirror to update replayers
|
||||
for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
|
||||
sleep ${s}
|
||||
test_image_replay_state "${daemon}" "${cluster}" "${image}" "${state}" && return 0
|
||||
test_image_replay_state "${daemon}" "${image}" "${state}" && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
@ -210,19 +311,17 @@ wait_for_image_replay_state()
|
||||
wait_for_image_replay_started()
|
||||
{
|
||||
local daemon=$1
|
||||
local cluster=$2
|
||||
local image=$3
|
||||
local image=$2
|
||||
|
||||
wait_for_image_replay_state "${daemon}" "${cluster}" "${image}" started
|
||||
wait_for_image_replay_state "${daemon}" "${image}" started
|
||||
}
|
||||
|
||||
wait_for_image_replay_stopped()
|
||||
{
|
||||
local daemon=$1
|
||||
local cluster=$2
|
||||
local image=$3
|
||||
local image=$2
|
||||
|
||||
wait_for_image_replay_state "${daemon}" "${cluster}" "${image}" stopped
|
||||
wait_for_image_replay_state "${daemon}" "${image}" stopped
|
||||
}
|
||||
|
||||
get_position()
|
||||
@ -234,7 +333,7 @@ get_position()
|
||||
# Parse line like below, looking for the first position
|
||||
# [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=9], [object_number=0, tag_tid=3, entry_tid=8], [object_number=3, tag_tid=3, entry_tid=7], [object_number=2, tag_tid=3, entry_tid=6]]]]
|
||||
|
||||
local status_log=${TEMPDIR}/${RMT_CLUSTER}-${POOL}-${image}.status
|
||||
local status_log=${TEMPDIR}/${CLUSTER2}-${POOL}-${image}.status
|
||||
rbd --cluster ${cluster} -p ${POOL} journal status --image ${image} |
|
||||
tee ${status_log} >&2
|
||||
sed -nEe 's/^.*\[id='"${id_regexp}"',.*positions=\[\[([^]]*)\],.*$/\1/p' \
|
||||
@ -283,20 +382,6 @@ create_image()
|
||||
--image-feature exclusive-lock --image-feature journaling ${image}
|
||||
}
|
||||
|
||||
create_remote_image()
|
||||
{
|
||||
local image=$1
|
||||
|
||||
create_image ${RMT_CLUSTER} ${image}
|
||||
}
|
||||
|
||||
create_local_image()
|
||||
{
|
||||
local image=$1
|
||||
|
||||
create_image ${LOC_CLUSTER} ${image}
|
||||
}
|
||||
|
||||
write_image()
|
||||
{
|
||||
local cluster=$1
|
||||
@ -312,12 +397,12 @@ compare_images()
|
||||
{
|
||||
local image=$1
|
||||
|
||||
local rmt_export=${TEMPDIR}/${RMT_CLUSTER}-${POOL}-${image}.export
|
||||
local loc_export=${TEMPDIR}/${LOC_CLUSTER}-${POOL}-${image}.export
|
||||
local rmt_export=${TEMPDIR}/${CLUSTER2}-${POOL}-${image}.export
|
||||
local loc_export=${TEMPDIR}/${CLUSTER1}-${POOL}-${image}.export
|
||||
|
||||
rm -f ${rmt_export} ${loc_export}
|
||||
rbd --cluster ${RMT_CLUSTER} -p ${POOL} export ${image} ${rmt_export}
|
||||
rbd --cluster ${LOC_CLUSTER} -p ${POOL} export ${image} ${loc_export}
|
||||
rbd --cluster ${CLUSTER2} -p ${POOL} export ${image} ${rmt_export}
|
||||
rbd --cluster ${CLUSTER1} -p ${POOL} export ${image} ${loc_export}
|
||||
cmp ${rmt_export} ${loc_export}
|
||||
}
|
||||
|
||||
@ -341,19 +426,18 @@ promote_image()
|
||||
# Main
|
||||
#
|
||||
|
||||
if [ "$1" = clean ]; then
|
||||
TEMPDIR=$2
|
||||
|
||||
if [ -z "${TEMPDIR}" -a -n "${RBD_MIRROR_TEMDIR}" ]; then
|
||||
TEMPDIR="${RBD_MIRROR_TEMDIR}"
|
||||
if [ "$#" -gt 0 ]
|
||||
then
|
||||
if [ -z "${RBD_MIRROR_TEMDIR}" ]
|
||||
then
|
||||
echo "RBD_MIRROR_TEMDIR is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
test -n "${TEMPDIR}"
|
||||
|
||||
RBD_MIRROR_NOCLEANUP=
|
||||
|
||||
cleanup
|
||||
exit
|
||||
TEMPDIR="${RBD_MIRROR_TEMDIR}"
|
||||
cd ${TEMPDIR}
|
||||
$@
|
||||
exit $?
|
||||
fi
|
||||
|
||||
set -xe
|
||||
@ -361,47 +445,48 @@ set -xe
|
||||
setup
|
||||
|
||||
echo "TEST: add image and test replay"
|
||||
start_mirror ${CLUSTER1_DAEMON} ${CLUSTER1}
|
||||
image=test
|
||||
create_remote_image ${image}
|
||||
wait_for_image_replay_started ${LOC_DAEMON} ${LOC_CLUSTER} ${image}
|
||||
write_image ${RMT_CLUSTER} ${image} 100
|
||||
wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image}
|
||||
create_image ${CLUSTER2} ${image}
|
||||
wait_for_image_replay_started ${CLUSTER1_DAEMON} ${image}
|
||||
write_image ${CLUSTER2} ${image} 100
|
||||
wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image}
|
||||
compare_images ${image}
|
||||
|
||||
echo "TEST: stop mirror, add image, start mirror and test replay"
|
||||
stop_mirror ${LOC_DAEMON}
|
||||
stop_mirror ${CLUSTER1_DAEMON}
|
||||
image1=test1
|
||||
create_remote_image ${image1}
|
||||
write_image ${RMT_CLUSTER} ${image1} 100
|
||||
start_mirror ${LOC_DAEMON} ${LOC_CLUSTER}
|
||||
wait_for_image_replay_started ${LOC_DAEMON} ${LOC_CLUSTER} ${image1}
|
||||
wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image1}
|
||||
create_image ${CLUSTER2} ${image1}
|
||||
write_image ${CLUSTER2} ${image1} 100
|
||||
start_mirror ${CLUSTER1_DAEMON} ${CLUSTER1}
|
||||
wait_for_image_replay_started ${CLUSTER1_DAEMON} ${image1}
|
||||
wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image1}
|
||||
compare_images ${image1}
|
||||
|
||||
echo "TEST: test the first image is replaying after restart"
|
||||
write_image ${RMT_CLUSTER} ${image} 100
|
||||
wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image}
|
||||
write_image ${CLUSTER2} ${image} 100
|
||||
wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image}
|
||||
compare_images ${image}
|
||||
|
||||
echo "TEST: failover and failback"
|
||||
start_mirror ${RMT_DAEMON} ${RMT_CLUSTER}
|
||||
start_mirror ${CLUSTER2_DAEMON} ${CLUSTER2}
|
||||
|
||||
# failover
|
||||
demote_image ${RMT_CLUSTER} ${image}
|
||||
wait_for_image_replay_stopped ${LOC_DAEMON} ${LOC_CLUSTER} ${image}
|
||||
promote_image ${LOC_CLUSTER} ${image}
|
||||
wait_for_image_replay_started ${RMT_DAEMON} ${RMT_CLUSTER} ${image}
|
||||
write_image ${LOC_CLUSTER} ${image} 100
|
||||
wait_for_replay_complete ${RMT_DAEMON} ${LOC_CLUSTER} ${image}
|
||||
demote_image ${CLUSTER2} ${image}
|
||||
wait_for_image_replay_stopped ${CLUSTER1_DAEMON} ${image}
|
||||
promote_image ${CLUSTER1} ${image}
|
||||
wait_for_image_replay_started ${CLUSTER2_DAEMON} ${image}
|
||||
write_image ${CLUSTER1} ${image} 100
|
||||
wait_for_replay_complete ${CLUSTER2_DAEMON} ${CLUSTER1} ${image}
|
||||
compare_images ${image}
|
||||
|
||||
# failback
|
||||
demote_image ${LOC_CLUSTER} ${image}
|
||||
wait_for_image_replay_stopped ${RMT_DAEMON} ${RMT_CLUSTER} ${image}
|
||||
promote_image ${RMT_CLUSTER} ${image}
|
||||
wait_for_image_replay_started ${LOC_DAEMON} ${LOC_CLUSTER} ${image}
|
||||
write_image ${RMT_CLUSTER} ${image} 100
|
||||
wait_for_replay_complete ${LOC_DAEMON} ${RMT_CLUSTER} ${image}
|
||||
demote_image ${CLUSTER1} ${image}
|
||||
wait_for_image_replay_stopped ${CLUSTER2_DAEMON} ${image}
|
||||
promote_image ${CLUSTER2} ${image}
|
||||
wait_for_image_replay_started ${CLUSTER1_DAEMON} ${image}
|
||||
write_image ${CLUSTER2} ${image} 100
|
||||
wait_for_replay_complete ${CLUSTER1_DAEMON} ${CLUSTER2} ${image}
|
||||
compare_images ${image}
|
||||
|
||||
echo OK
|
||||
|
||||
Loading…
Reference in New Issue
Block a user