diff --git a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml index 5f12b22399b..4fe1d6c10a5 100644 --- a/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml +++ b/qa/suites/rbd/mirror-thrash/workloads/rbd-mirror-journal-workunit.yaml @@ -1,12 +1,13 @@ meta: -- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_journal.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: "journal" RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml index 29047a77d79..f5edab17910 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-exclusive-lock.yaml @@ -1,12 +1,13 @@ meta: -- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'snapshot' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_CONFIG_KEY: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml index af13c92b575..e41596681fa 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-fast-diff.yaml @@ -1,12 +1,13 @@ meta: -- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'snapshot' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_CONFIG_KEY: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml index 5ea2bb105f0..ee6fc41ae8c 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-journaling.yaml @@ -1,12 +1,13 @@ meta: -- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'snapshot' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_CONFIG_KEY: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml index e21d57b2b68..6bb17151b80 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-snapshot-workunit-minimum.yaml @@ -1,12 +1,13 @@ meta: -- desc: run the rbd_mirror_snapshot.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in snapshot mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_snapshot.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'snapshot' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_CONFIG_KEY: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml index 0102050eb00..e5fbfd5610b 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-config-key.yaml @@ -1,12 +1,13 @@ meta: -- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_journal.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'journal' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_CONFIG_KEY: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml index 5bd024d2da8..2a1d06358b2 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-min-compat-client-octopus.yaml @@ -1,11 +1,12 @@ meta: -- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_journal.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'journal' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_MIN_COMPAT_CLIENT: 'octopus' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml index 0a610ea2f01..cba93ad3afc 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-none.yaml @@ -1,5 +1,5 @@ meta: -- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode overrides: ceph: conf: @@ -8,9 +8,10 @@ overrides: tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_journal.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'journal' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' diff --git a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml index 883e8abd31e..5c7d8486bdd 100644 --- a/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml +++ b/qa/suites/rbd/mirror/workloads/rbd-mirror-workunit-policy-simple.yaml @@ -1,5 +1,5 @@ meta: -- desc: run the rbd_mirror_journal.sh workunit to test the rbd-mirror daemon +- desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode overrides: ceph: conf: @@ -8,9 +8,10 @@ overrides: tasks: - workunit: clients: - cluster1.client.mirror: [rbd/rbd_mirror_journal.sh] + cluster1.client.mirror: [rbd/rbd_mirror.sh] env: # override workunit setting of CEPH_ARGS='--cluster' CEPH_ARGS: '' + RBD_MIRROR_MODE: 'journal' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror.sh similarity index 69% rename from qa/workunits/rbd/rbd_mirror_journal.sh rename to qa/workunits/rbd/rbd_mirror.sh index 54f6aeec8e0..bc3237dfc08 100755 --- a/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/qa/workunits/rbd/rbd_mirror.sh @@ -1,26 +1,42 @@ #!/bin/sh -ex # -# rbd_mirror_journal.sh - test rbd-mirror daemon in journal-based mirroring mode +# rbd_mirror.sh - test rbd-mirror daemon in snapshot or journal mirroring mode +# +# Usage: +# RBD_MIRROR_MODE=journal rbd_mirror.sh +# +# Use environment variable RBD_MIRROR_MODE to set the mode +# Available modes: snapshot | journal # # The scripts starts two ("local" and "remote") clusters using mstart.sh script, # creates a temporary directory, used for cluster configs, daemon logs, admin # socket, temporary files, and launches rbd-mirror daemon. # -. $(dirname $0)/rbd_mirror_helpers.sh +if [ "${#}" -gt 0 ]; then + echo "unnecessary arguments: ${@}" + exit 100 +fi +if [ "${RBD_MIRROR_MODE}" != "snapshot" ] && [ "${RBD_MIRROR_MODE}" != "journal" ]; then + echo "unknown mode: ${RBD_MIRROR_MODE}" + echo "set RBD_MIRROR_MODE env variable, available modes: snapshot | journal" + exit 100 +fi + +. $(dirname $0)/rbd_mirror_helpers.sh setup testlog "TEST: add image and test replay" start_mirrors ${CLUSTER1} image=test -create_image ${CLUSTER2} ${POOL} ${image} +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' fi @@ -31,12 +47,12 @@ compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" testlog "TEST: stop mirror, add image, start mirror and test replay" stop_mirrors ${CLUSTER1} image1=test1 -create_image ${CLUSTER2} ${POOL} ${image1} +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1} ${RBD_MIRROR_MODE} write_image ${CLUSTER2} ${POOL} ${image1} 100 start_mirrors ${CLUSTER1} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' fi @@ -46,7 +62,7 @@ testlog "TEST: test the first image is replaying after restart" write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} compare_images ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then @@ -124,6 +140,9 @@ remove_image_retry ${CLUSTER2} ${POOL} ${image1} testlog "TEST: test image rename" new_name="${image}_RENAMED" rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${POOL} ${new_name} +fi wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name} @@ -131,13 +150,16 @@ admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${POOL} ${image} +fi wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} testlog "TEST: test trash move restore" image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image}) trash_move ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' -trash_restore ${CLUSTER2} ${POOL} ${image_id} +trash_restore ${CLUSTER2} ${POOL} ${image} ${image_id} ${RBD_MIRROR_MODE} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)" @@ -146,7 +168,7 @@ remove_image_retry ${CLUSTER2} ${POOL} ${image} wait_for_image_in_omap ${CLUSTER1} ${POOL} wait_for_image_in_omap ${CLUSTER2} ${POOL} -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} @@ -165,7 +187,7 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} compare_images ${POOL} ${image} # failover (unmodified) @@ -184,7 +206,7 @@ wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' compare_images ${POOL} ${image} @@ -198,7 +220,7 @@ wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} write_image ${CLUSTER1} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} compare_images ${POOL} ${image} # failback @@ -210,7 +232,7 @@ promote_image ${CLUSTER2} ${POOL} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} write_image ${CLUSTER2} ${POOL} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' compare_images ${POOL} ${image} @@ -235,15 +257,20 @@ for i in `seq 1 20`; do wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' done +# check that demote (or other mirror snapshots) don't pile up +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3 + test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3 +fi testlog "TEST: force promote" force_promote_image=test_force_promote -create_image ${CLUSTER2} ${POOL} ${force_promote_image} +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} ${RBD_MIRROR_MODE} write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image} @@ -259,7 +286,7 @@ testlog "TEST: cloned images" testlog " - default" parent_image=test_parent parent_snap=snap -create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} +create_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${RBD_MIRROR_MODE} write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} @@ -267,24 +294,29 @@ protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} clone_image=test_clone clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image} write_image ${CLUSTER2} ${POOL} ${clone_image} 100 - -enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} journal +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + enable_mirror ${CLUSTER2} ${POOL} ${clone_image} ${RBD_MIRROR_MODE} +else + enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${RBD_MIRROR_MODE} +fi wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} compare_images ${PARENT_POOL} ${parent_image} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} compare_images ${POOL} ${clone_image} remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} testlog " - clone v1" -clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1 - -clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ - ${clone_image}_v1 --rbd-default-clone-format 1 +clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} \ + ${parent_image} ${parent_snap} ${POOL} ${clone_image}1 \ + ${RBD_MIRROR_MODE} +clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} \ + ${parent_image} ${parent_snap} ${POOL} ${clone_image}_v1 \ + ${RBD_MIRROR_MODE} --rbd-default-clone-format 1 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 @@ -296,13 +328,20 @@ remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} testlog " - clone v2" parent_snap=snap_v2 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ - ${clone_image}_v2 --rbd-default-clone-format 2 +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +fi +clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} \ + ${parent_image} ${parent_snap} ${POOL} ${clone_image}_v2 \ + ${RBD_MIRROR_MODE} --rbd-default-clone-format 2 test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +fi test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2 @@ -312,11 +351,18 @@ wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${pa testlog " - clone v2 non-primary" create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +fi wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} -clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ - ${clone_image}_v2 --rbd-default-clone-format 2 +clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} \ + ${parent_image} ${parent_snap} ${POOL} ${clone_image}_v2 \ + ${RBD_MIRROR_MODE} --rbd-default-clone-format 2 remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} +fi wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} @@ -324,7 +370,8 @@ remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} testlog "TEST: data pool" dp_image=test_data_pool -create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL} +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${dp_image} \ + ${RBD_MIRROR_MODE} 128 --data-pool ${PARENT_POOL} data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image} @@ -335,7 +382,7 @@ write_image ${CLUSTER2} ${POOL} ${dp_image} 100 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' write_image ${CLUSTER2} ${POOL} ${dp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} compare_images ${POOL} ${dp_image}@snap1 compare_images ${POOL} ${dp_image}@snap2 compare_images ${POOL} ${dp_image} @@ -347,7 +394,7 @@ image3=test3 image4=test4 image5=test5 for i in ${image2} ${image3} ${image4} ${image5}; do - create_image ${CLUSTER2} ${POOL} ${i} + create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${i} ${RBD_MIRROR_MODE} write_image ${CLUSTER2} ${POOL} ${i} 100 create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' @@ -356,6 +403,9 @@ for i in ${image2} ${image3} ${image4} ${image5}; do protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' fi write_image ${CLUSTER2} ${POOL} ${i} 100 + if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${POOL} ${i} + fi wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' done @@ -377,87 +427,100 @@ for i in ${image2} ${image3} ${image4} ${image5}; do wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' done -set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' -for i in ${image2} ${image4}; do - enable_journaling ${CLUSTER2} ${POOL} ${i} - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' - wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i} - compare_images ${POOL} ${i} -done +if [ "${RBD_MIRROR_MODE}" = "journal" ]; then + set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' + for i in ${image2} ${image4}; do + enable_journaling ${CLUSTER2} ${POOL} ${i} + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' + wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i} + compare_images ${POOL} ${i} + done -testlog "TEST: remove mirroring pool" -pool=pool_to_remove -for cluster in ${CLUSTER1} ${CLUSTER2}; do + testlog "TEST: remove mirroring pool" + pool=pool_to_remove + for cluster in ${CLUSTER1} ${CLUSTER2}; do CEPH_ARGS='' ceph --cluster ${cluster} osd pool create ${pool} 16 16 CEPH_ARGS='' rbd --cluster ${cluster} pool init ${pool} rbd --cluster ${cluster} mirror pool enable ${pool} pool -done -peer_add ${CLUSTER1} ${pool} ${CLUSTER2} -peer_add ${CLUSTER2} ${pool} ${CLUSTER1} -rdp_image=test_remove_data_pool -create_image ${CLUSTER2} ${pool} ${image} 128 -create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool} -write_image ${CLUSTER2} ${pool} ${image} 100 -write_image ${CLUSTER2} ${POOL} ${rdp_image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${pool} ${image} 'up+replaying' 'primary_position' -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} 'up+replaying' 'primary_position' -for cluster in ${CLUSTER1} ${CLUSTER2}; do + done + peer_add ${CLUSTER1} ${pool} ${CLUSTER2} + peer_add ${CLUSTER2} ${pool} ${CLUSTER1} + rdp_image=test_remove_data_pool + create_image ${CLUSTER2} ${pool} ${image} 128 + create_image ${CLUSTER2} ${POOL} ${rdp_image} 128 --data-pool ${pool} + write_image ${CLUSTER2} ${pool} ${image} 100 + write_image ${CLUSTER2} ${POOL} ${rdp_image} 100 + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${pool} ${image} + wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${pool} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${rdp_image} + wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${rdp_image} + for cluster in ${CLUSTER1} ${CLUSTER2}; do CEPH_ARGS='' ceph --cluster ${cluster} osd pool rm ${pool} ${pool} --yes-i-really-really-mean-it -done -remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image} -wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted' -for i in 0 1 2 4 8 8 8 8 16 16; do + done + remove_image_retry ${CLUSTER2} ${POOL} ${rdp_image} + wait_for_image_present ${CLUSTER1} ${POOL} ${rdp_image} 'deleted' + for i in 0 1 2 4 8 8 8 8 16 16; do sleep $i admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} || break -done -admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false + done + admin_daemons "${CLUSTER2}" rbd mirror status ${pool}/${image} && false +fi testlog "TEST: snapshot rename" snap_name='snap_rename' +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + enable_mirror ${CLUSTER2} ${POOL} ${image2} +fi create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0" for i in `seq 1 20`; do rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}" done +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2} +fi wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1' unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2' for i in ${image2} ${image4}; do - remove_image_retry ${CLUSTER2} ${POOL} ${i} + remove_image_retry ${CLUSTER2} ${POOL} ${i} done testlog "TEST: disable mirror while daemon is stopped" stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} -set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' +if [ "${RBD_MIRROR_MODE}" = "journal" ]; then + set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' +fi disable_mirror ${CLUSTER2} ${POOL} ${image} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' -set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' -enable_journaling ${CLUSTER2} ${POOL} ${image} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + enable_mirror ${CLUSTER2} ${POOL} ${image} +else + set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' + enable_journaling ${CLUSTER2} ${POOL} ${image} +fi wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} testlog "TEST: non-default namespace image mirroring" testlog " - replay" -create_image ${CLUSTER2} ${POOL}/${NS1} ${image} -create_image ${CLUSTER2} ${POOL}/${NS2} ${image} -enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} journal +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image} ${RBD_MIRROR_MODE} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} ${RBD_MIRROR_MODE} wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' 'primary_position' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} compare_images ${POOL}/${NS1} ${image} compare_images ${POOL}/${NS2} ${image} @@ -470,7 +533,7 @@ remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} testlog " - data pool" dp_image=test_data_pool -create_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 128 --data-pool ${PARENT_POOL} +create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} ${RBD_MIRROR_MODE} 128 --data-pool ${PARENT_POOL} data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} @@ -478,7 +541,7 @@ data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) test "${data_pool}" = "${PARENT_POOL}" write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} compare_images ${POOL}/${NS1} ${dp_image} remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} @@ -487,11 +550,11 @@ request_resync_image ${CLUSTER1} ${POOL} ${image} image_id wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} compare_images ${POOL} ${image} -testlog "TEST: image resync while replayer is stopped" if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + testlog "TEST: image resync while replayer is stopped" admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} request_resync_image ${CLUSTER1} ${POOL} ${image} image_id @@ -500,7 +563,7 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' + wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} compare_images ${POOL} ${image} fi @@ -511,86 +574,88 @@ start_mirrors ${CLUSTER1} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} compare_images ${POOL} ${image} remove_image_retry ${CLUSTER2} ${POOL} ${image} -testlog "TEST: client disconnect" -image=laggy -create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K -write_image ${CLUSTER2} ${POOL} ${image} 10 +if [ "${RBD_MIRROR_MODE}" = "journal" ]; then + testlog "TEST: client disconnect" + image=laggy + create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K + write_image ${CLUSTER2} ${POOL} ${image} 10 -testlog " - replay stopped after disconnect" -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" -disconnect_image ${CLUSTER2} ${POOL} ${image} -test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' - -testlog " - replay started after resync requested" -request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" -compare_images ${POOL} ${image} - -testlog " - disconnected after max_concurrent_object_sets reached" -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + testlog " - replay stopped after disconnect" + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" - set_image_meta ${CLUSTER2} ${POOL} ${image} \ - conf_rbd_journal_max_concurrent_object_sets 1 - write_image ${CLUSTER2} ${POOL} ${image} 20 16384 - write_image ${CLUSTER2} ${POOL} ${image} 20 16384 + disconnect_image ${CLUSTER2} ${POOL} ${image} test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" - set_image_meta ${CLUSTER2} ${POOL} ${image} \ - conf_rbd_journal_max_concurrent_object_sets 0 - - testlog " - replay is still stopped (disconnected) after restart" - admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' + + testlog " - replay started after resync requested" + request_resync_image ${CLUSTER1} ${POOL} ${image} image_id + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + compare_images ${POOL} ${image} + + testlog " - disconnected after max_concurrent_object_sets reached" + if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then + admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_journal_max_concurrent_object_sets 1 + write_image ${CLUSTER2} ${POOL} ${image} 20 16384 + write_image ${CLUSTER2} ${POOL} ${image} 20 16384 + test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_journal_max_concurrent_object_sets 0 + + testlog " - replay is still stopped (disconnected) after restart" + admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' + fi + + testlog " - replay started after resync requested" + request_resync_image ${CLUSTER1} ${POOL} ${image} image_id + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + compare_images ${POOL} ${image} + + testlog " - rbd_mirroring_resync_after_disconnect config option" + set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_mirroring_resync_after_disconnect true + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image}) + disconnect_image ${CLUSTER2} ${POOL} ${image} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} + wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' + wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + compare_images ${POOL} ${image} + set_image_meta ${CLUSTER2} ${POOL} ${image} \ + conf_rbd_mirroring_resync_after_disconnect false + wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} + disconnect_image ${CLUSTER2} ${POOL} ${image} + test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" + wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} + wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' + remove_image_retry ${CLUSTER2} ${POOL} ${image} fi -testlog " - replay started after resync requested" -request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" -compare_images ${POOL} ${image} - -testlog " - rbd_mirroring_resync_after_disconnect config option" -set_image_meta ${CLUSTER2} ${POOL} ${image} \ - conf_rbd_mirroring_resync_after_disconnect true -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -image_id=$(get_image_id ${CLUSTER1} ${POOL} ${image}) -disconnect_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -test -n "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" -compare_images ${POOL} ${image} -set_image_meta ${CLUSTER2} ${POOL} ${image} \ - conf_rbd_mirroring_resync_after_disconnect false -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -disconnect_image ${CLUSTER2} ${POOL} ${image} -test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' -remove_image_retry ${CLUSTER2} ${POOL} ${image} - testlog "TEST: split-brain" image=split-brain -create_image ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} ${RBD_MIRROR_MODE} +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} promote_image ${CLUSTER1} ${POOL} ${image} --force wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' @@ -598,7 +663,7 @@ write_image ${CLUSTER1} ${POOL} ${image} 10 demote_image ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +wait_for_replaying_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} remove_image_retry ${CLUSTER2} ${POOL} ${image} testlog "TEST: check if removed images' OMAP are removed" diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index f4961b925e6..7183a2cba25 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -24,7 +24,7 @@ # The cleanup can be done as a separate step, running the script with # `cleanup ${RBD_MIRROR_TEMDIR}' arguments. # -# Note, as other workunits tests, rbd_mirror_journal.sh expects to find ceph binaries +# Note, as other workunits tests, rbd_mirror_helpers.sh expects to find ceph binaries # in PATH. # # Thus a typical troubleshooting session: @@ -35,7 +35,7 @@ # cd $CEPH_SRC_PATH # PATH=$CEPH_SRC_PATH:$PATH # RBD_MIRROR_NOCLEANUP=1 RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ -# ../qa/workunits/rbd/rbd_mirror_journal.sh +# RBD_MIRROR_MODE=journal ../qa/workunits/rbd/rbd_mirror.sh # # After the test failure cd to TEMPDIR and check the current state: # @@ -49,21 +49,27 @@ # ceph --admin-daemon rbd-mirror.cluster1_daemon.cluster1.$pid.asok help # ... # +# To setup the environment without actually running the tests: +# +# cd $CEPH_SRC_PATH +# RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ +# ../qa/workunits/rbd_mirror_helpers.sh setup +# # Also you can execute commands (functions) from the script: # # cd $CEPH_SRC_PATH # export RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror -# ../qa/workunits/rbd/rbd_mirror_journal.sh status -# ../qa/workunits/rbd/rbd_mirror_journal.sh stop_mirror cluster1 -# ../qa/workunits/rbd/rbd_mirror_journal.sh start_mirror cluster2 -# ../qa/workunits/rbd/rbd_mirror_journal.sh flush cluster2 +# ../qa/workunits/rbd/rbd_mirror_helpers.sh status +# ../qa/workunits/rbd/rbd_mirror_helpers.sh stop_mirror cluster1 +# ../qa/workunits/rbd/rbd_mirror_helpers.sh start_mirror cluster2 +# ../qa/workunits/rbd/rbd_mirror_helpers.sh flush cluster2 # ... # # Eventually, run the cleanup: # # cd $CEPH_SRC_PATH # RBD_MIRROR_TEMDIR=/tmp/tmp.rbd_mirror \ -# ../qa/workunits/rbd/rbd_mirror_journal.sh cleanup +# ../qa/workunits/rbd_mirror_helpers.sh cleanup # if type xmlstarlet > /dev/null 2>&1; then @@ -90,6 +96,10 @@ RBD_IMAGE_FEATURES=${RBD_IMAGE_FEATURES:-layering,exclusive-lock,journaling} MIRROR_USER_ID_PREFIX=${MIRROR_USER_ID_PREFIX:-${CEPH_ID}.} MIRROR_POOL_MODE=${MIRROR_POOL_MODE:-pool} MIRROR_IMAGE_MODE=${MIRROR_IMAGE_MODE:-journal} +if [ "${RBD_MIRROR_MODE}" = "snapshot" ]; then + MIRROR_POOL_MODE=image + MIRROR_IMAGE_MODE=snapshot +fi export CEPH_ARGS="--id ${CEPH_ID}" @@ -880,6 +890,20 @@ wait_for_status_in_pool_dir() return 1 } +wait_for_replaying_status_in_pool_dir() +{ + local cluster=$1 + local pool=$2 + local image=$3 + + if [ "${RBD_MIRROR_MODE}" = "journal" ]; then + wait_for_status_in_pool_dir ${cluster} ${pool} ${image} 'up+replaying' \ + 'primary_position' + else + wait_for_status_in_pool_dir ${cluster} ${pool} ${image} 'up+replaying' + fi +} + create_image() { local cluster=$1 ; shift @@ -896,6 +920,23 @@ create_image() --image-feature "${RBD_IMAGE_FEATURES}" $@ ${pool}/${image} } +is_pool_mirror_mode_image() +{ + local pool=$1 + + if [ "${MIRROR_POOL_MODE}" = "image" ]; then + return 0 + fi + + case "${pool}" in + */${NS2} | ${PARENT_POOL}) + return 0 + ;; + esac + + return 1 +} + create_image_and_enable_mirror() { local cluster=$1 ; shift @@ -907,7 +948,7 @@ create_image_and_enable_mirror() fi create_image ${cluster} ${pool} ${image} $@ - if [ "${MIRROR_POOL_MODE}" = "image" ] || [ "$pool" = "${PARENT_POOL}" ]; then + if is_pool_mirror_mode_image ${pool}; then enable_mirror ${cluster} ${pool} ${image} ${mode} fi } @@ -987,9 +1028,14 @@ trash_move() { trash_restore() { local cluster=$1 local pool=$2 - local image_id=$3 + local image=$3 + local image_id=$4 + local mode=${5:-${RBD_MIRROR_MODE}} rbd --cluster=${cluster} trash restore ${pool}/${image_id} + if is_pool_mirror_mode_image ${pool}; then + enable_mirror ${cluster} ${pool} ${image} ${mode} + fi } clone_image() @@ -1024,7 +1070,9 @@ clone_image_and_enable_mirror() fi clone_image ${cluster} ${parent_pool} ${parent_image} ${parent_snap} ${clone_pool} ${clone_image} $@ - enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode} + if is_pool_mirror_mode_image ${clone_pool}; then + enable_mirror ${cluster} ${clone_pool} ${clone_image} ${mode} + fi } disconnect_image() diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh deleted file mode 100755 index c70d48b09db..00000000000 --- a/qa/workunits/rbd/rbd_mirror_snapshot.sh +++ /dev/null @@ -1,517 +0,0 @@ -#!/bin/sh -ex -# -# rbd_mirror_snapshot.sh - test rbd-mirror daemon in snapshot-based mirroring mode -# -# The scripts starts two ("local" and "remote") clusters using mstart.sh script, -# creates a temporary directory, used for cluster configs, daemon logs, admin -# socket, temporary files, and launches rbd-mirror daemon. -# - -MIRROR_POOL_MODE=image -MIRROR_IMAGE_MODE=snapshot - -. $(dirname $0)/rbd_mirror_helpers.sh - -setup - -testlog "TEST: add image and test replay" -start_mirrors ${CLUSTER1} -image=test -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} -set_image_meta ${CLUSTER2} ${POOL} ${image} "key1" "value1" -set_image_meta ${CLUSTER2} ${POOL} ${image} "key2" "value2" -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown' -fi -compare_images ${POOL} ${image} -compare_image_meta ${CLUSTER1} ${POOL} ${image} "key1" "value1" -compare_image_meta ${CLUSTER1} ${POOL} ${image} "key2" "value2" - -testlog "TEST: stop mirror, add image, start mirror and test replay" -stop_mirrors ${CLUSTER1} -image1=test1 -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image1} -write_image ${CLUSTER2} ${POOL} ${image1} 100 -start_mirrors ${CLUSTER1} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown' -fi -compare_images ${POOL} ${image1} - -testlog "TEST: test the first image is replaying after restart" -write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -compare_images ${POOL} ${image} - -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - testlog "TEST: stop/start/restart mirror via admin socket" - all_admin_daemons ${CLUSTER1} rbd mirror stop - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - - all_admin_daemons ${CLUSTER1} rbd mirror start - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - - all_admin_daemons ${CLUSTER1} rbd mirror restart - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - - all_admin_daemons ${CLUSTER1} rbd mirror stop - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - - all_admin_daemons ${CLUSTER1} rbd mirror restart - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - - all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - - admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - - all_admin_daemons ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - - admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - - all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - - all_admin_daemons ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped' - - all_admin_daemons ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}${PEER_CLUSTER_SUFFIX} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' - - flush ${CLUSTER1} - all_admin_daemons ${CLUSTER1} rbd mirror status -fi - -remove_image_retry ${CLUSTER2} ${POOL} ${image1} - -testlog "TEST: test image rename" -new_name="${image}_RENAMED" -rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} -mirror_image_snapshot ${CLUSTER2} ${POOL} ${new_name} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' -admin_daemons ${CLUSTER1} rbd mirror status ${POOL}/${new_name} -admin_daemons ${CLUSTER1} rbd mirror restart ${POOL}/${new_name} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying' -rename_image ${CLUSTER2} ${POOL} ${new_name} ${image} -mirror_image_snapshot ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - -testlog "TEST: test trash move restore" -image_id=$(get_image_id ${CLUSTER2} ${POOL} ${image}) -trash_move ${CLUSTER2} ${POOL} ${image} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' -trash_restore ${CLUSTER2} ${POOL} ${image_id} -enable_mirror ${CLUSTER2} ${POOL} ${image} snapshot -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - -testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)" -remove_image_retry ${CLUSTER2} ${POOL} ${image} - -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} - -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - -testlog "TEST: failover and failback" -start_mirrors ${CLUSTER2} - -# demote and promote same cluster -demote_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' -promote_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -compare_images ${POOL} ${image} - -# failover (unmodified) -demote_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' -promote_image ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} - -# failback (unmodified) -demote_image ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' -promote_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -compare_images ${POOL} ${image} - -# failover -demote_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' -promote_image ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} -write_image ${CLUSTER1} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' -compare_images ${POOL} ${image} - -# failback -demote_image ${CLUSTER1} ${POOL} ${image} -wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' -promote_image ${CLUSTER2} ${POOL} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -write_image ${CLUSTER2} ${POOL} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' -compare_images ${POOL} ${image} - -testlog "TEST: failover / failback loop" -for i in `seq 1 20`; do - demote_image ${CLUSTER2} ${POOL} ${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' - wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' - promote_image ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' - wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' - demote_image ${CLUSTER1} ${POOL} ${image} - wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown' - wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown' - promote_image ${CLUSTER2} ${POOL} ${image} - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped' - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -done -# check that demote (or other mirror snapshots) don't pile up -test "$(count_mirror_snaps ${CLUSTER1} ${POOL} ${image})" -le 3 -test "$(count_mirror_snaps ${CLUSTER2} ${POOL} ${image})" -le 3 - -testlog "TEST: force promote" -force_promote_image=test_force_promote -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${force_promote_image} -write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 -wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' -promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force' -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image} -wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped' -wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' -write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100 -write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 -remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image} -remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image} - -testlog "TEST: cloned images" -testlog " - default" -parent_image=test_parent -parent_snap=snap -create_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} -write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100 -create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} - -clone_image=test_clone -clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image} -write_image ${CLUSTER2} ${POOL} ${clone_image} 100 -enable_mirror ${CLUSTER2} ${POOL} ${clone_image} snapshot - -wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' -compare_images ${PARENT_POOL} ${parent_image} - -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' -compare_images ${POOL} ${clone_image} -remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} - -testlog " - clone v1" -clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ - ${parent_snap} ${POOL} ${clone_image}1 - -clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ - ${parent_snap} ${POOL} ${clone_image}_v1 snapshot --rbd-default-clone-format 1 -test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 -test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 -remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1 -remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1 -unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} - -testlog " - clone v2" -parent_snap=snap_v2 -create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} -clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ - ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2 -test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v2) = 2 -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v2 -test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v2) = 2 - -remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} -test_snap_moved_to_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} -remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v2 -wait_for_image_present ${CLUSTER1} ${POOL} ${clone_image}_v2 'deleted' -test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} - -testlog " - clone v2 non-primary" -create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} -wait_for_snap_present ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} -clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ - ${parent_snap} ${POOL} ${clone_image}_v2 snapshot --rbd-default-clone-format 2 -remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} -mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} -wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} -remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 -wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} -remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} - -testlog "TEST: data pool" -dp_image=test_data_pool -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL} -data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image}) -test "${data_pool}" = "${PARENT_POOL}" -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image} -data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image}) -test "${data_pool}" = "${PARENT_POOL}" -create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1' -write_image ${CLUSTER2} ${POOL} ${dp_image} 100 -create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2' -write_image ${CLUSTER2} ${POOL} ${dp_image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' -compare_images ${POOL} ${dp_image}@snap1 -compare_images ${POOL} ${dp_image}@snap2 -compare_images ${POOL} ${dp_image} -remove_image_retry ${CLUSTER2} ${POOL} ${dp_image} - -testlog "TEST: disable mirroring / delete non-primary image" -image2=test2 -image3=test3 -image4=test4 -image5=test5 -for i in ${image2} ${image3} ${image4} ${image5}; do - create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${i} - write_image ${CLUSTER2} ${POOL} ${i} 100 - create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' - create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' - if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then - protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' - protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' - fi - write_image ${CLUSTER2} ${POOL} ${i} 100 - mirror_image_snapshot ${CLUSTER2} ${POOL} ${i} - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present' - wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2' -done - -set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' -for i in ${image2} ${image4}; do - disable_mirror ${CLUSTER2} ${POOL} ${i} -done - -unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1' -unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2' -for i in ${image3} ${image5}; do - remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1' - remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2' - remove_image_retry ${CLUSTER2} ${POOL} ${i} -done - -for i in ${image2} ${image3} ${image4} ${image5}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done - -testlog "TEST: snapshot rename" -snap_name='snap_rename' -enable_mirror ${CLUSTER2} ${POOL} ${image2} -create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0" -for i in `seq 1 20`; do - rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}" -done -mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2} -wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" - -unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1' -unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2' -for i in ${image2} ${image4}; do - remove_image_retry ${CLUSTER2} ${POOL} ${i} -done - -testlog "TEST: disable mirror while daemon is stopped" -stop_mirrors ${CLUSTER1} -stop_mirrors ${CLUSTER2} -disable_mirror ${CLUSTER2} ${POOL} ${image} -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - test_image_present ${CLUSTER1} ${POOL} ${image} 'present' -fi -start_mirrors ${CLUSTER1} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' -enable_mirror ${CLUSTER2} ${POOL} ${image} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - -testlog "TEST: non-default namespace image mirroring" -testlog " - replay" -create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${image} -create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${image} -wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS2} ${image} -write_image ${CLUSTER2} ${POOL}/${NS1} ${image} 100 -write_image ${CLUSTER2} ${POOL}/${NS2} ${image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${image} -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS2} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${image} 'up+replaying' -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS2} ${image} 'up+replaying' -compare_images ${POOL}/${NS1} ${image} -compare_images ${POOL}/${NS2} ${image} - -testlog " - disable mirroring / delete image" -remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} -disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} -wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' -wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' -remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} - -testlog " - data pool" -dp_image=test_data_pool -create_image_and_enable_mirror ${CLUSTER2} ${POOL}/${NS1} ${dp_image} snapshot 128 --data-pool ${PARENT_POOL} -data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL}/${NS1} ${dp_image}) -test "${data_pool}" = "${PARENT_POOL}" -wait_for_image_replay_started ${CLUSTER1} ${POOL}/${NS1} ${dp_image} -data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL}/${NS1} ${dp_image}) -test "${data_pool}" = "${PARENT_POOL}" -write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 -wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' -compare_images ${POOL}/${NS1} ${dp_image} -remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} - -testlog "TEST: simple image resync" -request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -compare_images ${POOL} ${image} - -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - testlog "TEST: image resync while replayer is stopped" - admin_daemons ${CLUSTER1} rbd mirror stop ${POOL}/${image} - wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} - request_resync_image ${CLUSTER1} ${POOL} ${image} image_id - admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} - wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} - admin_daemons ${CLUSTER1} rbd mirror start ${POOL}/${image} - wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' - wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} - wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' - compare_images ${POOL} ${image} -fi - -testlog "TEST: request image resync while daemon is offline" -stop_mirrors ${CLUSTER1} -request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -start_mirrors ${CLUSTER1} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id} -wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' -wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -compare_images ${POOL} ${image} -remove_image_retry ${CLUSTER2} ${POOL} ${image} - -testlog "TEST: split-brain" -image=split-brain -create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -promote_image ${CLUSTER1} ${POOL} ${image} --force -wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped' -write_image ${CLUSTER1} ${POOL} ${image} 10 -demote_image ${CLUSTER1} ${POOL} ${image} -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' -request_resync_image ${CLUSTER1} ${POOL} ${image} image_id -wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' -remove_image_retry ${CLUSTER2} ${POOL} ${image} - -testlog "TEST: check if removed images' OMAP are removed" -start_mirrors ${CLUSTER2} -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} - -if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then - # teuthology will trash the daemon - testlog "TEST: no blocklists" - CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blocklist ls 2>&1 | grep -q "listed 0 entries" - CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blocklist ls 2>&1 | grep -q "listed 0 entries" -fi