mirror of
https://github.com/ceph/ceph
synced 2025-03-11 02:39:05 +00:00
Merge pull request #30411 from dillaman/wip-41653
rbd-mirror: simplify peer bootstrapping Reviewed-by: Mykola Golub <mgolub@suse.com>
This commit is contained in:
commit
9068510305
@ -39,14 +39,14 @@ tasks to configure mirroring using the ``rbd`` command. Mirroring is
|
||||
configured on a per-pool basis within the Ceph clusters.
|
||||
|
||||
The pool configuration steps should be performed on both peer clusters. These
|
||||
procedures assume two clusters, named "local" and "remote", are accessible from
|
||||
procedures assume two clusters, named "site-a" and "site-b", are accessible from
|
||||
a single host for clarity.
|
||||
|
||||
See the `rbd`_ manpage for additional details of how to connect to different
|
||||
Ceph clusters.
|
||||
|
||||
.. note:: The cluster name in the following examples corresponds to a Ceph
|
||||
configuration file of the same name (e.g. /etc/ceph/remote.conf). See the
|
||||
configuration file of the same name (e.g. /etc/ceph/site-b.conf). See the
|
||||
`ceph-conf`_ documentation for how to configure multiple clusters.
|
||||
|
||||
Enable Mirroring
|
||||
@ -66,8 +66,8 @@ The mirroring mode can either be ``pool`` or ``image``:
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror pool enable image-pool pool
|
||||
$ rbd --cluster remote mirror pool enable image-pool pool
|
||||
$ rbd --cluster site-a mirror pool enable image-pool pool
|
||||
$ rbd --cluster site-b mirror pool enable image-pool pool
|
||||
|
||||
Disable Mirroring
|
||||
-----------------
|
||||
@ -83,23 +83,72 @@ explicitly.
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror pool disable image-pool
|
||||
$ rbd --cluster remote mirror pool disable image-pool
|
||||
$ rbd --cluster site-a mirror pool disable image-pool
|
||||
$ rbd --cluster site-b mirror pool disable image-pool
|
||||
|
||||
Add Cluster Peer
|
||||
----------------
|
||||
Bootstrap Peers
|
||||
---------------
|
||||
|
||||
In order for the ``rbd-mirror`` daemon to discover its peer cluster, the peer
|
||||
needs to be registered to the pool. To add a mirroring peer Ceph cluster with
|
||||
``rbd``, specify the ``mirror pool peer add`` command, the pool name, and a
|
||||
cluster specification::
|
||||
needs to be registered to the pool and a user account needs to be created.
|
||||
This process can be automated with ``rbd`` and the
|
||||
``mirror pool peer bootstrap create`` and ``mirror pool peer bootstrap import``
|
||||
commands.
|
||||
|
||||
To manually create a new bootstrap token with ``rbd``, specify the
|
||||
``mirror pool peer bootstrap create`` command, a pool name, along with an
|
||||
optional friendly site name to describe the local cluster::
|
||||
|
||||
rbd mirror pool peer bootstrap create [--site-name {local-site-name}] {pool-name}
|
||||
|
||||
The output of ``mirror pool peer bootstrap create`` will be a token that should
|
||||
be provided to the ``mirror pool peer bootstrap import`` command. For example,
|
||||
on site-a::
|
||||
|
||||
$ rbd --cluster site-a mirror pool peer bootstrap create --site-name site-a image-pool
|
||||
eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0ifQ==
|
||||
|
||||
To manually import the bootstrap token created by another cluster with ``rbd``,
|
||||
specify the ``mirror pool peer bootstrap import`` command, the pool name, a file
|
||||
path to the created token (or '-' to read from standard input), along with an
|
||||
optional friendly site name to describe the local cluster and a mirroring
|
||||
direction (defaults to rx-tx for bidirectional mirroring, but can also be set
|
||||
to rx-only for unidirectional mirroring)::
|
||||
|
||||
rbd mirror pool peer bootstrap import [--site-name {local-site-name}] [--direction {rx-only or rx-tx}] {pool-name} {token-path}
|
||||
|
||||
For example, on site-b::
|
||||
|
||||
$ cat <<EOF > token
|
||||
eyJmc2lkIjoiOWY1MjgyZGItYjg5OS00NTk2LTgwOTgtMzIwYzFmYzM5NmYzIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFBUnczOWQwdkhvQmhBQVlMM1I4RmR5dHNJQU50bkFTZ0lOTVE9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMS4zOjY4MjAsdjE6MTkyLjE2OC4xLjM6NjgyMV0ifQ==
|
||||
EOF
|
||||
$ rbd --cluster site-b mirror pool peer bootstrap import --site-name site-b image-pool token
|
||||
|
||||
Add Cluster Peer Manually
|
||||
-------------------------
|
||||
|
||||
Cluster peers can be specified manually if desired or if the above bootstrap
|
||||
commands are not available with the currently installed Ceph release.
|
||||
|
||||
The remote ``rbd-mirror`` daemon will need access to the local cluster to
|
||||
perform mirroring. A new local Ceph user should be created for the remote
|
||||
daemon to use. To `create a Ceph user`_, with ``ceph`` specify the
|
||||
``auth get-or-create`` command, user name, monitor caps, and OSD caps::
|
||||
|
||||
ceph auth get-or-create client.rbd-mirror-peer mon 'profile rbd' osd 'profile rbd'
|
||||
|
||||
The resulting keyring should be copied to the other cluster's ``rbd-mirror``
|
||||
daemon hosts if not using the Ceph monitor ``config-key`` store described below.
|
||||
|
||||
To manually add a mirroring peer Ceph cluster with ``rbd``, specify the
|
||||
``mirror pool peer add`` command, the pool name, and a cluster specification::
|
||||
|
||||
rbd mirror pool peer add {pool-name} {client-name}@{cluster-name}
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror pool peer add image-pool client.remote@remote
|
||||
$ rbd --cluster remote mirror pool peer add image-pool client.local@local
|
||||
$ rbd --cluster site-a mirror pool peer add image-pool client.rbd-mirror-peer@site-b
|
||||
$ rbd --cluster site-b mirror pool peer add image-pool client.rbd-mirror-peer@site-a
|
||||
|
||||
By default, the ``rbd-mirror`` daemon needs to have access to a Ceph
|
||||
configuration file located at ``/etc/ceph/{cluster-name}.conf`` that provides
|
||||
@ -112,12 +161,15 @@ stored within the local Ceph monitor ``config-key`` store. To specify the
|
||||
peer cluster connection attributes when adding a mirroring peer, use the
|
||||
``--remote-mon-host`` and ``--remote-key-file`` optionals. For example::
|
||||
|
||||
$ rbd --cluster local mirror pool peer add image-pool client.remote@remote --remote-mon-host 192.168.1.1,192.168.1.2 --remote-key-file <(echo 'AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw==')
|
||||
$ rbd --cluster local mirror pool info image-pool --all
|
||||
$ cat <<EOF > remote-key-file
|
||||
AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw==
|
||||
EOF
|
||||
$ rbd --cluster site-a mirror pool peer add image-pool client.rbd-mirror-peer@site-b --remote-mon-host 192.168.1.1,192.168.1.2 --remote-key-file remote-key-file
|
||||
$ rbd --cluster site-a mirror pool info image-pool --all
|
||||
Mode: pool
|
||||
Peers:
|
||||
UUID NAME CLIENT MON_HOST KEY
|
||||
587b08db-3d33-4f32-8af8-421e77abb081 remote client.remote 192.168.1.1,192.168.1.2 AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw==
|
||||
UUID NAME CLIENT MON_HOST KEY
|
||||
587b08db-3d33-4f32-8af8-421e77abb081 site-b client.rbd-mirror-peer 192.168.1.1,192.168.1.2 AQAeuZdbMMoBChAAcj++/XUxNOLFaWdtTREEsw==
|
||||
|
||||
Remove Cluster Peer
|
||||
-------------------
|
||||
@ -130,8 +182,8 @@ To remove a mirroring peer Ceph cluster with ``rbd``, specify the
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror pool peer remove image-pool 55672766-c02b-4729-8567-f13a66893445
|
||||
$ rbd --cluster remote mirror pool peer remove image-pool 60c0e299-b38f-4234-91f6-eed0a367be08
|
||||
$ rbd --cluster site-a mirror pool peer remove image-pool 55672766-c02b-4729-8567-f13a66893445
|
||||
$ rbd --cluster site-b mirror pool peer remove image-pool 60c0e299-b38f-4234-91f6-eed0a367be08
|
||||
|
||||
Data Pools
|
||||
----------
|
||||
@ -177,7 +229,7 @@ the ``feature enable`` command, the pool and image name, and the feature name::
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local feature enable image-pool/image-1 journaling
|
||||
$ rbd --cluster site-a feature enable image-pool/image-1 journaling
|
||||
|
||||
.. note:: The journaling feature is dependent on the exclusive-lock feature. If
|
||||
the exclusive-lock feature is not already enabled, it should be enabled prior
|
||||
@ -198,7 +250,7 @@ To enable mirroring for a specific image with ``rbd``, specify the
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror image enable image-pool/image-1
|
||||
$ rbd --cluster site-a mirror image enable image-pool/image-1
|
||||
|
||||
Disable Image Mirroring
|
||||
-----------------------
|
||||
@ -210,7 +262,7 @@ To disable mirroring for a specific image with ``rbd``, specify the
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror image disable image-pool/image-1
|
||||
$ rbd --cluster site-a mirror image disable image-pool/image-1
|
||||
|
||||
Image Promotion and Demotion
|
||||
----------------------------
|
||||
@ -232,7 +284,7 @@ To demote a specific image to non-primary with ``rbd``, specify the
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror image demote image-pool/image-1
|
||||
$ rbd --cluster site-a mirror image demote image-pool/image-1
|
||||
|
||||
To demote all primary images within a pool to non-primary with ``rbd``, specify
|
||||
the ``mirror pool demote`` command along with the pool name::
|
||||
@ -241,7 +293,7 @@ the ``mirror pool demote`` command along with the pool name::
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror pool demote image-pool
|
||||
$ rbd --cluster site-a mirror pool demote image-pool
|
||||
|
||||
To promote a specific image to primary with ``rbd``, specify the
|
||||
``mirror image promote`` command along with the pool and image name::
|
||||
@ -250,7 +302,7 @@ To promote a specific image to primary with ``rbd``, specify the
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster remote mirror image promote image-pool/image-1
|
||||
$ rbd --cluster site-b mirror image promote image-pool/image-1
|
||||
|
||||
To promote all non-primary images within a pool to primary with ``rbd``, specify
|
||||
the ``mirror pool promote`` command along with the pool name::
|
||||
@ -259,7 +311,7 @@ the ``mirror pool promote`` command along with the pool name::
|
||||
|
||||
For example::
|
||||
|
||||
$ rbd --cluster local mirror pool promote image-pool
|
||||
$ rbd --cluster site-a mirror pool promote image-pool
|
||||
|
||||
.. tip:: Since the primary / non-primary status is per-image, it is possible to
|
||||
have two clusters split the IO load and stage failover / failback.
|
||||
|
@ -0,0 +1,11 @@
|
||||
meta:
|
||||
- desc: run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon
|
||||
tasks:
|
||||
- workunit:
|
||||
clients:
|
||||
cluster1.client.mirror: [rbd/rbd_mirror_bootstrap.sh]
|
||||
env:
|
||||
# override workunit setting of CEPH_ARGS='--cluster'
|
||||
CEPH_ARGS: ''
|
||||
RBD_MIRROR_INSTANCES: '1'
|
||||
RBD_MIRROR_USE_EXISTING_CLUSTER: '1'
|
49
qa/workunits/rbd/rbd_mirror_bootstrap.sh
Executable file
49
qa/workunits/rbd/rbd_mirror_bootstrap.sh
Executable file
@ -0,0 +1,49 @@
|
||||
#!/bin/sh -ex
|
||||
#
|
||||
# rbd_mirror_bootstrap.sh - test peer bootstrap create/import
|
||||
#
|
||||
|
||||
RBD_MIRROR_MANUAL_PEERS=1
|
||||
RBD_MIRROR_INSTANCES=${RBD_MIRROR_INSTANCES:-1}
|
||||
. $(dirname $0)/rbd_mirror_helpers.sh
|
||||
|
||||
setup
|
||||
|
||||
testlog "TEST: bootstrap cluster2 from cluster1"
|
||||
# create token on cluster1 and import to cluster2
|
||||
TOKEN=${TEMPDIR}/peer-token
|
||||
TOKEN_2=${TEMPDIR}/peer-token-2
|
||||
rbd --cluster ${CLUSTER1} mirror pool peer bootstrap create ${POOL} > ${TOKEN}
|
||||
rbd --cluster ${CLUSTER1} mirror pool peer bootstrap create ${PARENT_POOL} > ${TOKEN_2}
|
||||
cmp ${TOKEN} ${TOKEN_2}
|
||||
|
||||
rbd --cluster ${CLUSTER2} --pool ${POOL} mirror pool peer bootstrap import ${TOKEN} --direction rx-only
|
||||
rbd --cluster ${CLUSTER2} --pool ${PARENT_POOL} mirror pool peer bootstrap import ${TOKEN} --direction rx-tx
|
||||
|
||||
start_mirrors ${CLUSTER1}
|
||||
start_mirrors ${CLUSTER2}
|
||||
|
||||
testlog "TEST: verify rx-only direction"
|
||||
[ "$(rbd --cluster ${CLUSTER1} --pool ${POOL} mirror pool info --format xml |
|
||||
${XMLSTARLET} sel -t -v '//mirror/peers/peer[1]/uuid')" = "" ]
|
||||
|
||||
create_image ${CLUSTER1} ${POOL} image1
|
||||
|
||||
wait_for_image_replay_started ${CLUSTER2} ${POOL} image1
|
||||
write_image ${CLUSTER1} ${POOL} image1 100
|
||||
wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} image1
|
||||
|
||||
testlog "TEST: verify rx-tx direction"
|
||||
create_image ${CLUSTER1} ${PARENT_POOL} image1
|
||||
create_image ${CLUSTER2} ${PARENT_POOL} image2
|
||||
|
||||
enable_mirror ${CLUSTER1} ${PARENT_POOL} image1
|
||||
enable_mirror ${CLUSTER2} ${PARENT_POOL} image2
|
||||
|
||||
wait_for_image_replay_started ${CLUSTER2} ${PARENT_POOL} image1
|
||||
write_image ${CLUSTER1} ${PARENT_POOL} image1 100
|
||||
wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${PARENT_POOL} image1
|
||||
|
||||
wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} image2
|
||||
write_image ${CLUSTER2} ${PARENT_POOL} image2 100
|
||||
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} image2
|
@ -268,26 +268,28 @@ setup_pools()
|
||||
rbd --cluster ${cluster} mirror pool enable ${POOL}/ns1 pool
|
||||
rbd --cluster ${cluster} mirror pool enable ${POOL}/ns2 image
|
||||
|
||||
if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then
|
||||
rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster}
|
||||
rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster}
|
||||
else
|
||||
mon_map_file=${TEMPDIR}/${remote_cluster}.monmap
|
||||
ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file}
|
||||
mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' |
|
||||
head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g')
|
||||
if [ -z ${RBD_MIRROR_MANUAL_PEERS} ]; then
|
||||
if [ -z ${RBD_MIRROR_CONFIG_KEY} ]; then
|
||||
rbd --cluster ${cluster} mirror pool peer add ${POOL} ${remote_cluster}
|
||||
rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} ${remote_cluster}
|
||||
else
|
||||
mon_map_file=${TEMPDIR}/${remote_cluster}.monmap
|
||||
ceph --cluster ${remote_cluster} mon getmap > ${mon_map_file}
|
||||
mon_addr=$(monmaptool --print ${mon_map_file} | grep -E 'mon\.' |
|
||||
head -n 1 | sed -E 's/^[0-9]+: ([^ ]+).+$/\1/' | sed -E 's/\/[0-9]+//g')
|
||||
|
||||
admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key
|
||||
CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file}
|
||||
admin_key_file=${TEMPDIR}/${remote_cluster}.client.${CEPH_ID}.key
|
||||
CEPH_ARGS='' ceph --cluster ${remote_cluster} auth get-key client.${CEPH_ID} > ${admin_key_file}
|
||||
|
||||
rbd --cluster ${cluster} mirror pool peer add ${POOL} client.${CEPH_ID}@${remote_cluster}-DNE \
|
||||
--remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file}
|
||||
rbd --cluster ${cluster} mirror pool peer add ${POOL} client.${CEPH_ID}@${remote_cluster}-DNE \
|
||||
--remote-mon-host "${mon_addr}" --remote-key-file ${admin_key_file}
|
||||
|
||||
uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}-DNE)
|
||||
rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr}
|
||||
rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file}
|
||||
uuid=$(rbd --cluster ${cluster} mirror pool peer add ${PARENT_POOL} client.${CEPH_ID}@${remote_cluster}-DNE)
|
||||
rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} mon-host ${mon_addr}
|
||||
rbd --cluster ${cluster} mirror pool peer set ${PARENT_POOL} ${uuid} key-file ${admin_key_file}
|
||||
|
||||
PEER_CLUSTER_SUFFIX=-DNE
|
||||
PEER_CLUSTER_SUFFIX=-DNE
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
|
@ -1323,6 +1323,7 @@ inline namespace v14_2_0 {
|
||||
Rados();
|
||||
explicit Rados(IoCtx& ioctx);
|
||||
~Rados();
|
||||
static void from_rados_t(rados_t cluster, Rados &rados);
|
||||
|
||||
int init(const char * const id);
|
||||
int init2(const char * const name, const char * const clustername,
|
||||
|
@ -139,6 +139,12 @@ typedef enum {
|
||||
RBD_MIRROR_MODE_POOL /* mirroring enabled on all journaled images */
|
||||
} rbd_mirror_mode_t;
|
||||
|
||||
typedef enum {
|
||||
RBD_MIRROR_PEER_DIRECTION_RX = 0,
|
||||
RBD_MIRROR_PEER_DIRECTION_TX = 1,
|
||||
RBD_MIRROR_PEER_DIRECTION_RX_TX = 2
|
||||
} rbd_mirror_peer_direction_t;
|
||||
|
||||
typedef struct {
|
||||
char *uuid;
|
||||
char *cluster_name;
|
||||
@ -431,10 +437,22 @@ CEPH_RBD_API void rbd_migration_status_cleanup(
|
||||
rbd_image_migration_status_t *status);
|
||||
|
||||
/* pool mirroring */
|
||||
CEPH_RBD_API int rbd_mirror_site_name_get(rados_t cluster,
|
||||
char *name, size_t *max_len);
|
||||
CEPH_RBD_API int rbd_mirror_site_name_set(rados_t cluster,
|
||||
const char *name);
|
||||
|
||||
CEPH_RBD_API int rbd_mirror_mode_get(rados_ioctx_t io_ctx,
|
||||
rbd_mirror_mode_t *mirror_mode);
|
||||
CEPH_RBD_API int rbd_mirror_mode_set(rados_ioctx_t io_ctx,
|
||||
rbd_mirror_mode_t mirror_mode);
|
||||
|
||||
CEPH_RBD_API int rbd_mirror_peer_bootstrap_create(rados_ioctx_t io_ctx,
|
||||
char *token, size_t *max_len);
|
||||
CEPH_RBD_API int rbd_mirror_peer_bootstrap_import(
|
||||
rados_ioctx_t io_ctx, rbd_mirror_peer_direction_t direction,
|
||||
const char *token);
|
||||
|
||||
CEPH_RBD_API int rbd_mirror_peer_add(rados_ioctx_t io_ctx,
|
||||
char *uuid, size_t uuid_max_length,
|
||||
const char *cluster_name,
|
||||
|
@ -74,6 +74,8 @@ namespace librbd {
|
||||
std::string address;
|
||||
} locker_t;
|
||||
|
||||
typedef rbd_mirror_peer_direction_t mirror_peer_direction_t;
|
||||
|
||||
typedef struct {
|
||||
std::string uuid;
|
||||
std::string cluster_name;
|
||||
@ -263,8 +265,18 @@ public:
|
||||
image_migration_status_t *status, size_t status_size);
|
||||
|
||||
// RBD pool mirroring support functions
|
||||
int mirror_site_name_get(librados::Rados& rados, std::string* site_name);
|
||||
int mirror_site_name_set(librados::Rados& rados,
|
||||
const std::string& site_name);
|
||||
|
||||
int mirror_mode_get(IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode);
|
||||
int mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode);
|
||||
|
||||
int mirror_peer_bootstrap_create(IoCtx& io_ctx, std::string* token);
|
||||
int mirror_peer_bootstrap_import(IoCtx& io_ctx,
|
||||
mirror_peer_direction_t direction,
|
||||
const std::string &token);
|
||||
|
||||
int mirror_peer_add(IoCtx& io_ctx, std::string *uuid,
|
||||
const std::string &cluster_name,
|
||||
const std::string &client_name);
|
||||
|
@ -116,7 +116,10 @@
|
||||
* MON config-key prefix for storing optional remote cluster connectivity
|
||||
* parameters
|
||||
*/
|
||||
#define RBD_MIRROR_PEER_CONFIG_KEY_PREFIX "rbd/mirror/peer/"
|
||||
#define RBD_MIRROR_CONFIG_KEY_PREFIX "rbd/mirror/"
|
||||
#define RBD_MIRROR_SITE_NAME_CONFIG_KEY RBD_MIRROR_CONFIG_KEY_PREFIX "site_name"
|
||||
#define RBD_MIRROR_PEER_CLIENT_ID_CONFIG_KEY RBD_MIRROR_CONFIG_KEY_PREFIX "peer_client_id"
|
||||
#define RBD_MIRROR_PEER_CONFIG_KEY_PREFIX RBD_MIRROR_CONFIG_KEY_PREFIX "peer/"
|
||||
|
||||
struct rbd_info {
|
||||
ceph_le64 max_id;
|
||||
|
@ -2270,6 +2270,16 @@ librados::Rados::~Rados()
|
||||
shutdown();
|
||||
}
|
||||
|
||||
void librados::Rados::from_rados_t(rados_t cluster, Rados &rados) {
|
||||
if (rados.client) {
|
||||
rados.client->put();
|
||||
}
|
||||
rados.client = static_cast<RadosClient*>(cluster);
|
||||
if (rados.client) {
|
||||
rados.client->get();
|
||||
}
|
||||
}
|
||||
|
||||
int librados::Rados::init(const char * const id)
|
||||
{
|
||||
return rados_create((rados_t *)&client, id);
|
||||
|
@ -23,6 +23,8 @@
|
||||
#include "librbd/mirror/PromoteRequest.h"
|
||||
#include "librbd/mirror/Types.h"
|
||||
#include "librbd/MirroringWatcher.h"
|
||||
#include <boost/algorithm/string/trim.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <boost/scope_exit.hpp>
|
||||
#include "json_spirit/json_spirit.h"
|
||||
|
||||
@ -35,6 +37,56 @@ namespace api {
|
||||
|
||||
namespace {
|
||||
|
||||
int get_config_key(librados::Rados& rados, const std::string& key,
|
||||
std::string* value) {
|
||||
std::string cmd =
|
||||
"{"
|
||||
"\"prefix\": \"config-key get\", "
|
||||
"\"key\": \"" + key + "\""
|
||||
"}";
|
||||
|
||||
bufferlist in_bl;
|
||||
bufferlist out_bl;
|
||||
|
||||
int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr);
|
||||
if (r == -EINVAL) {
|
||||
return -EOPNOTSUPP;
|
||||
} else if (r < 0 && r != -ENOENT) {
|
||||
return r;
|
||||
}
|
||||
|
||||
*value = out_bl.to_str();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int set_config_key(librados::Rados& rados, const std::string& key,
|
||||
const std::string& value) {
|
||||
std::string cmd;
|
||||
if (value.empty()) {
|
||||
cmd = "{"
|
||||
"\"prefix\": \"config-key rm\", "
|
||||
"\"key\": \"" + key + "\""
|
||||
"}";
|
||||
} else {
|
||||
cmd = "{"
|
||||
"\"prefix\": \"config-key set\", "
|
||||
"\"key\": \"" + key + "\", "
|
||||
"\"val\": \"" + value + "\""
|
||||
"}";
|
||||
}
|
||||
bufferlist in_bl;
|
||||
bufferlist out_bl;
|
||||
|
||||
int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr);
|
||||
if (r == -EINVAL) {
|
||||
return -EOPNOTSUPP;
|
||||
} else if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string get_peer_config_key_name(int64_t pool_id,
|
||||
const std::string& peer_uuid) {
|
||||
return RBD_MIRROR_PEER_CONFIG_KEY_PREFIX + stringify(pool_id) + "/" +
|
||||
@ -44,22 +96,151 @@ std::string get_peer_config_key_name(int64_t pool_id,
|
||||
int remove_peer_config_key(librados::IoCtx& io_ctx,
|
||||
const std::string& peer_uuid) {
|
||||
int64_t pool_id = io_ctx.get_id();
|
||||
std::string cmd =
|
||||
"{"
|
||||
"\"prefix\": \"config-key rm\", "
|
||||
"\"key\": \"" + get_peer_config_key_name(pool_id, peer_uuid) + "\""
|
||||
"}";
|
||||
auto key = get_peer_config_key_name(pool_id, peer_uuid);
|
||||
|
||||
bufferlist in_bl;
|
||||
bufferlist out_bl;
|
||||
librados::Rados rados(io_ctx);
|
||||
int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr);
|
||||
int r = set_config_key(rados, key, "");
|
||||
if (r < 0 && r != -ENOENT && r != -EPERM) {
|
||||
return r;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int create_bootstrap_user(CephContext* cct, librados::Rados& rados,
|
||||
std::string* peer_client_id, std::string* cephx_key) {
|
||||
ldout(cct, 20) << dendl;
|
||||
|
||||
// retrieve peer CephX user from config-key
|
||||
int r = get_config_key(rados, RBD_MIRROR_PEER_CLIENT_ID_CONFIG_KEY,
|
||||
peer_client_id);
|
||||
if (r == -EACCES) {
|
||||
ldout(cct, 5) << "insufficient permissions to get peer-client-id "
|
||||
<< "config-key" << dendl;
|
||||
return r;
|
||||
} else if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to retrieve peer client id key: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
} else if (r == -ENOENT || peer_client_id->empty()) {
|
||||
ldout(cct, 20) << "creating new peer-client-id config-key" << dendl;
|
||||
|
||||
*peer_client_id = "rbd-mirror-peer";
|
||||
r = set_config_key(rados, RBD_MIRROR_PEER_CLIENT_ID_CONFIG_KEY,
|
||||
*peer_client_id);
|
||||
if (r == -EACCES) {
|
||||
ldout(cct, 5) << "insufficient permissions to update peer-client-id "
|
||||
<< "config-key" << dendl;
|
||||
return r;
|
||||
} else if (r < 0) {
|
||||
lderr(cct) << "failed to update peer client id key: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
}
|
||||
ldout(cct, 20) << "peer_client_id=" << *peer_client_id << dendl;
|
||||
|
||||
// create peer client user
|
||||
std::string cmd =
|
||||
R"({)" \
|
||||
R"( "prefix": "auth get-or-create",)" \
|
||||
R"( "entity": "client.)" + *peer_client_id + R"(",)" \
|
||||
R"( "caps": [)" \
|
||||
R"( "mon", "profile rbd-mirror-peer",)" \
|
||||
R"( "osd", "profile rbd"],)" \
|
||||
R"( "format": "json")" \
|
||||
R"(})";
|
||||
|
||||
bufferlist in_bl;
|
||||
bufferlist out_bl;
|
||||
|
||||
r = rados.mon_command(cmd, in_bl, &out_bl, nullptr);
|
||||
if (r == -EINVAL) {
|
||||
ldout(cct, 5) << "caps mismatch for existing user" << dendl;
|
||||
return -EEXIST;
|
||||
} else if (r == -EACCES) {
|
||||
ldout(cct, 5) << "insufficient permissions to create user" << dendl;
|
||||
return r;
|
||||
} else if (r < 0) {
|
||||
lderr(cct) << "failed to create or update RBD mirroring bootstrap user: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
// extract key from response
|
||||
bool json_valid = false;
|
||||
json_spirit::mValue json_root;
|
||||
if(json_spirit::read(out_bl.to_str(), json_root)) {
|
||||
try {
|
||||
auto& json_obj = json_root.get_array()[0].get_obj();
|
||||
*cephx_key = json_obj["key"].get_str();
|
||||
json_valid = true;
|
||||
} catch (std::runtime_error&) {
|
||||
}
|
||||
}
|
||||
|
||||
if (!json_valid) {
|
||||
lderr(cct) << "invalid auth keyring JSON received" << dendl;
|
||||
return -EBADMSG;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int create_bootstrap_peer(CephContext* cct, librados::IoCtx& io_ctx,
|
||||
const std::string& site_name, const std::string& fsid,
|
||||
const std::string& client_id, const std::string& key,
|
||||
const std::string& mon_host,
|
||||
const std::string& cluster1,
|
||||
const std::string& cluster2) {
|
||||
ldout(cct, 20) << dendl;
|
||||
|
||||
std::string peer_uuid;
|
||||
std::vector<mirror_peer_t> peers;
|
||||
int r = Mirror<>::peer_list(io_ctx, &peers);
|
||||
if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to list mirror peers: " << cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
if (peers.empty()) {
|
||||
r = Mirror<>::peer_add(io_ctx, &peer_uuid, site_name,
|
||||
"client." + client_id);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to add " << cluster1 << " peer to "
|
||||
<< cluster2 << " " << "cluster: " << cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
} else if (peers[0].cluster_name != site_name &&
|
||||
peers[0].cluster_name != fsid) {
|
||||
// only support a single peer
|
||||
lderr(cct) << "multiple peers are not currently supported" << dendl;
|
||||
return -EINVAL;
|
||||
} else {
|
||||
peer_uuid = peers[0].uuid;
|
||||
|
||||
if (peers[0].cluster_name != site_name) {
|
||||
r = Mirror<>::peer_set_cluster(io_ctx, peer_uuid, site_name);
|
||||
if (r < 0) {
|
||||
// non-fatal attempt to update site name
|
||||
lderr(cct) << "failed to update peer site name" << dendl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Mirror<>::Attributes attributes {
|
||||
{"mon_host", mon_host},
|
||||
{"key", key}};
|
||||
r = Mirror<>::peer_set_attributes(io_ctx, peer_uuid, attributes);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to update " << cluster1 << " cluster connection "
|
||||
<< "attributes in " << cluster2 << " cluster: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int validate_mirroring_enabled(I *ictx) {
|
||||
CephContext *cct = ictx->cct;
|
||||
@ -430,11 +611,6 @@ int Mirror<I>::image_resync(I *ictx) {
|
||||
return r;
|
||||
}
|
||||
|
||||
r = validate_mirroring_enabled(ictx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
C_SaferCond tag_owner_ctx;
|
||||
bool is_tag_owner;
|
||||
Journal<I>::is_tag_owner(ictx, &is_tag_owner, &tag_owner_ctx);
|
||||
@ -540,6 +716,51 @@ int Mirror<I>::image_get_instance_id(I *ictx, std::string *instance_id) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int Mirror<I>::site_name_get(librados::Rados& rados, std::string* name) {
|
||||
CephContext *cct = reinterpret_cast<CephContext *>(rados.cct());
|
||||
ldout(cct, 20) << dendl;
|
||||
|
||||
int r = get_config_key(rados, RBD_MIRROR_SITE_NAME_CONFIG_KEY, name);
|
||||
if (r == -EOPNOTSUPP) {
|
||||
return r;
|
||||
} else if (r == -ENOENT || name->empty()) {
|
||||
// default to the cluster fsid
|
||||
r = rados.cluster_fsid(name);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve cluster fsid: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
}
|
||||
return r;
|
||||
} else if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve site name: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int Mirror<I>::site_name_set(librados::Rados& rados, const std::string& name) {
|
||||
CephContext *cct = reinterpret_cast<CephContext *>(rados.cct());
|
||||
|
||||
std::string site_name{name};
|
||||
boost::algorithm::trim(site_name);
|
||||
ldout(cct, 20) << "site_name=" << site_name << dendl;
|
||||
|
||||
int r = set_config_key(rados, RBD_MIRROR_SITE_NAME_CONFIG_KEY, name);
|
||||
if (r == -EOPNOTSUPP) {
|
||||
return r;
|
||||
} else if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to update site name: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int Mirror<I>::mode_get(librados::IoCtx& io_ctx,
|
||||
rbd_mirror_mode_t *mirror_mode) {
|
||||
@ -763,6 +984,234 @@ int Mirror<I>::mode_set(librados::IoCtx& io_ctx,
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int Mirror<I>::peer_bootstrap_create(librados::IoCtx& io_ctx,
|
||||
std::string* token) {
|
||||
CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
|
||||
ldout(cct, 20) << dendl;
|
||||
|
||||
auto mirror_mode = cls::rbd::MIRROR_MODE_DISABLED;
|
||||
int r = cls_client::mirror_mode_get(&io_ctx, &mirror_mode);
|
||||
if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to retrieve mirroring mode: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
} else if (mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
// retrieve the cluster fsid
|
||||
std::string fsid;
|
||||
librados::Rados rados(io_ctx);
|
||||
r = rados.cluster_fsid(&fsid);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve cluster fsid: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string peer_client_id;
|
||||
std::string cephx_key;
|
||||
r = create_bootstrap_user(cct, rados, &peer_client_id, &cephx_key);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string mon_host = cct->_conf.get_val<std::string>("mon_host");
|
||||
ldout(cct, 20) << "mon_host=" << mon_host << dendl;
|
||||
|
||||
// format the token response
|
||||
bufferlist token_bl;
|
||||
token_bl.append(
|
||||
R"({)" \
|
||||
R"("fsid":")" + fsid + R"(",)" + \
|
||||
R"("client_id":")" + peer_client_id + R"(",)" + \
|
||||
R"("key":")" + cephx_key + R"(",)" + \
|
||||
R"("mon_host":")" + \
|
||||
boost::replace_all_copy(mon_host, "\"", "\\\"") + R"(")" + \
|
||||
R"(})");
|
||||
ldout(cct, 20) << "token=" << token_bl.to_str() << dendl;
|
||||
|
||||
bufferlist base64_bl;
|
||||
token_bl.encode_base64(base64_bl);
|
||||
*token = base64_bl.to_str();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int Mirror<I>::peer_bootstrap_import(librados::IoCtx& io_ctx,
|
||||
rbd_mirror_peer_direction_t direction,
|
||||
const std::string& token) {
|
||||
CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
|
||||
ldout(cct, 20) << dendl;
|
||||
|
||||
if (direction != RBD_MIRROR_PEER_DIRECTION_RX &&
|
||||
direction != RBD_MIRROR_PEER_DIRECTION_RX_TX) {
|
||||
lderr(cct) << "invalid mirror peer direction" << dendl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
bufferlist token_bl;
|
||||
try {
|
||||
bufferlist base64_bl;
|
||||
base64_bl.append(token);
|
||||
token_bl.decode_base64(base64_bl);
|
||||
} catch (buffer::error& err) {
|
||||
lderr(cct) << "failed to decode base64" << dendl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
ldout(cct, 20) << "token=" << token_bl.to_str() << dendl;
|
||||
|
||||
bool json_valid = false;
|
||||
std::string expected_remote_fsid;
|
||||
std::string remote_client_id;
|
||||
std::string remote_key;
|
||||
std::string remote_mon_host;
|
||||
|
||||
json_spirit::mValue json_root;
|
||||
if(json_spirit::read(token_bl.to_str(), json_root)) {
|
||||
try {
|
||||
auto& json_obj = json_root.get_obj();
|
||||
expected_remote_fsid = json_obj["fsid"].get_str();
|
||||
remote_client_id = json_obj["client_id"].get_str();
|
||||
remote_key = json_obj["key"].get_str();
|
||||
remote_mon_host = json_obj["mon_host"].get_str();
|
||||
json_valid = true;
|
||||
} catch (std::runtime_error&) {
|
||||
}
|
||||
}
|
||||
|
||||
if (!json_valid) {
|
||||
lderr(cct) << "invalid bootstrap token JSON received" << dendl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
// sanity check import process
|
||||
std::string local_fsid;
|
||||
librados::Rados rados(io_ctx);
|
||||
int r = rados.cluster_fsid(&local_fsid);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve cluster fsid: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string local_site_name;
|
||||
r = site_name_get(rados, &local_site_name);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve cluster site name: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
// attempt to connect to remote cluster
|
||||
librados::Rados remote_rados;
|
||||
remote_rados.init(remote_client_id.c_str());
|
||||
|
||||
auto remote_cct = reinterpret_cast<CephContext*>(remote_rados.cct());
|
||||
remote_cct->_conf.set_val("mon_host", remote_mon_host);
|
||||
remote_cct->_conf.set_val("key", remote_key);
|
||||
|
||||
r = remote_rados.connect();
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to connect to peer cluster: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string remote_fsid;
|
||||
r = remote_rados.cluster_fsid(&remote_fsid);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve remote cluster fsid: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
} else if (local_fsid == remote_fsid) {
|
||||
lderr(cct) << "cannot import token for local cluster" << dendl;
|
||||
return -EINVAL;
|
||||
} else if (expected_remote_fsid != remote_fsid) {
|
||||
lderr(cct) << "unexpected remote cluster fsid" << dendl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
std::string remote_site_name;
|
||||
r = site_name_get(remote_rados, &remote_site_name);
|
||||
if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve remote cluster site name: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
} else if (local_site_name == remote_site_name) {
|
||||
lderr(cct) << "cannot import token for duplicate site name" << dendl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
librados::IoCtx remote_io_ctx;
|
||||
r = remote_rados.ioctx_create(io_ctx.get_pool_name().c_str(), remote_io_ctx);
|
||||
if (r == -ENOENT) {
|
||||
ldout(cct, 10) << "remote pool does not exist" << dendl;
|
||||
return r;
|
||||
} else if (r < 0) {
|
||||
lderr(cct) << "failed to open remote pool '" << io_ctx.get_pool_name()
|
||||
<< "': " << cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
}
|
||||
|
||||
auto remote_mirror_mode = cls::rbd::MIRROR_MODE_DISABLED;
|
||||
r = cls_client::mirror_mode_get(&remote_io_ctx, &remote_mirror_mode);
|
||||
if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to retrieve remote mirroring mode: "
|
||||
<< cpp_strerror(r) << dendl;
|
||||
return r;
|
||||
} else if (remote_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
|
||||
return -ENOSYS;
|
||||
}
|
||||
|
||||
auto local_mirror_mode = cls::rbd::MIRROR_MODE_DISABLED;
|
||||
r = cls_client::mirror_mode_get(&io_ctx, &local_mirror_mode);
|
||||
if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to retrieve local mirroring mode: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
} else if (local_mirror_mode == cls::rbd::MIRROR_MODE_DISABLED) {
|
||||
// copy mirror mode from remote peer
|
||||
r = mode_set(io_ctx, static_cast<rbd_mirror_mode_t>(remote_mirror_mode));
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
if (direction == RBD_MIRROR_PEER_DIRECTION_RX_TX) {
|
||||
// create a local mirror peer user and export it to the remote cluster
|
||||
std::string local_client_id;
|
||||
std::string local_key;
|
||||
r = create_bootstrap_user(cct, rados, &local_client_id, &local_key);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string local_mon_host = cct->_conf.get_val<std::string>("mon_host");
|
||||
|
||||
// create local cluster peer in remote cluster
|
||||
r = create_bootstrap_peer(cct, remote_io_ctx, local_site_name, local_fsid,
|
||||
local_client_id, local_key, local_mon_host,
|
||||
"local", "remote");
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
// create remote cluster peer in local cluster
|
||||
r = create_bootstrap_peer(cct, io_ctx, remote_site_name, remote_fsid,
|
||||
remote_client_id, remote_key, remote_mon_host,
|
||||
"remote", "local");
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
template <typename I>
|
||||
int Mirror<I>::peer_add(librados::IoCtx& io_ctx, std::string *uuid,
|
||||
const std::string &cluster_name,
|
||||
@ -887,18 +1336,12 @@ int Mirror<I>::peer_get_attributes(librados::IoCtx& io_ctx,
|
||||
ldout(cct, 20) << "uuid=" << uuid << dendl;
|
||||
|
||||
attributes->clear();
|
||||
std::string cmd =
|
||||
"{"
|
||||
"\"prefix\": \"config-key get\", "
|
||||
"\"key\": \"" + get_peer_config_key_name(io_ctx.get_id(), uuid) + "\""
|
||||
"}";
|
||||
|
||||
bufferlist in_bl;
|
||||
bufferlist out_bl;
|
||||
|
||||
librados::Rados rados(io_ctx);
|
||||
int r = rados.mon_command(cmd, in_bl, &out_bl, nullptr);
|
||||
if (r == -ENOENT || out_bl.length() == 0) {
|
||||
std::string value;
|
||||
int r = get_config_key(rados, get_peer_config_key_name(io_ctx.get_id(), uuid),
|
||||
&value);
|
||||
if (r == -ENOENT || value.empty()) {
|
||||
return -ENOENT;
|
||||
} else if (r < 0) {
|
||||
lderr(cct) << "failed to retrieve peer attributes: " << cpp_strerror(r)
|
||||
@ -908,7 +1351,7 @@ int Mirror<I>::peer_get_attributes(librados::IoCtx& io_ctx,
|
||||
|
||||
bool json_valid = false;
|
||||
json_spirit::mValue json_root;
|
||||
if(json_spirit::read(out_bl.to_str(), json_root)) {
|
||||
if(json_spirit::read(value, json_root)) {
|
||||
try {
|
||||
auto& json_obj = json_root.get_obj();
|
||||
for (auto& pairs : json_obj) {
|
||||
@ -959,18 +1402,10 @@ int Mirror<I>::peer_set_attributes(librados::IoCtx& io_ctx,
|
||||
}
|
||||
ss << "}";
|
||||
|
||||
std::string cmd =
|
||||
"{"
|
||||
"\"prefix\": \"config-key set\", "
|
||||
"\"key\": \"" + get_peer_config_key_name(io_ctx.get_id(), uuid) + "\", "
|
||||
"\"val\": \"" + ss.str() + "\""
|
||||
"}";
|
||||
bufferlist in_bl;
|
||||
bufferlist out_bl;
|
||||
|
||||
librados::Rados rados(io_ctx);
|
||||
r = rados.mon_command(cmd, in_bl, &out_bl, nullptr);
|
||||
if (r < 0) {
|
||||
r = set_config_key(rados, get_peer_config_key_name(io_ctx.get_id(), uuid),
|
||||
ss.str());
|
||||
if (r < 0 && r != -ENOENT) {
|
||||
lderr(cct) << "failed to update peer attributes: " << cpp_strerror(r)
|
||||
<< dendl;
|
||||
return r;
|
||||
|
@ -23,9 +23,17 @@ struct Mirror {
|
||||
typedef std::map<std::string, mirror_image_status_t> IdToMirrorImageStatus;
|
||||
typedef std::map<mirror_image_status_state_t, int> MirrorImageStatusStates;
|
||||
|
||||
static int site_name_get(librados::Rados& rados, std::string* name);
|
||||
static int site_name_set(librados::Rados& rados, const std::string& name);
|
||||
|
||||
static int mode_get(librados::IoCtx& io_ctx, rbd_mirror_mode_t *mirror_mode);
|
||||
static int mode_set(librados::IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode);
|
||||
|
||||
static int peer_bootstrap_create(librados::IoCtx& io_ctx, std::string* token);
|
||||
static int peer_bootstrap_import(librados::IoCtx& io_ctx,
|
||||
rbd_mirror_peer_direction_t direction,
|
||||
const std::string& token);
|
||||
|
||||
static int peer_add(librados::IoCtx& io_ctx, std::string *uuid,
|
||||
const std::string &cluster_name,
|
||||
const std::string &client_name);
|
||||
|
@ -868,10 +868,31 @@ namespace librbd {
|
||||
return librbd::api::Mirror<>::mode_get(io_ctx, mirror_mode);
|
||||
}
|
||||
|
||||
int RBD::mirror_site_name_get(librados::Rados& rados,
|
||||
std::string* site_name) {
|
||||
return librbd::api::Mirror<>::site_name_get(rados, site_name);
|
||||
}
|
||||
|
||||
int RBD::mirror_site_name_set(librados::Rados& rados,
|
||||
const std::string& site_name) {
|
||||
return librbd::api::Mirror<>::site_name_set(rados, site_name);
|
||||
}
|
||||
|
||||
int RBD::mirror_mode_set(IoCtx& io_ctx, rbd_mirror_mode_t mirror_mode) {
|
||||
return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode);
|
||||
}
|
||||
|
||||
int RBD::mirror_peer_bootstrap_create(IoCtx& io_ctx, std::string* token) {
|
||||
return librbd::api::Mirror<>::peer_bootstrap_create(io_ctx, token);
|
||||
}
|
||||
|
||||
int RBD::mirror_peer_bootstrap_import(IoCtx& io_ctx,
|
||||
rbd_mirror_peer_direction_t direction,
|
||||
const std::string& token) {
|
||||
return librbd::api::Mirror<>::peer_bootstrap_import(io_ctx, direction,
|
||||
token);
|
||||
}
|
||||
|
||||
int RBD::mirror_peer_add(IoCtx& io_ctx, std::string *uuid,
|
||||
const std::string &cluster_name,
|
||||
const std::string &client_name) {
|
||||
@ -2718,6 +2739,34 @@ extern "C" int rbd_image_options_is_empty(rbd_image_options_t opts)
|
||||
}
|
||||
|
||||
/* pool mirroring */
|
||||
extern "C" int rbd_mirror_site_name_get(rados_t cluster, char *name,
|
||||
size_t *max_len) {
|
||||
librados::Rados rados;
|
||||
librados::Rados::from_rados_t(cluster, rados);
|
||||
|
||||
std::string site_name;
|
||||
int r = librbd::api::Mirror<>::site_name_get(rados, &site_name);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
auto total_len = site_name.size() + 1;
|
||||
if (*max_len < total_len) {
|
||||
*max_len = total_len;
|
||||
return -ERANGE;
|
||||
}
|
||||
*max_len = total_len;
|
||||
|
||||
strcpy(name, site_name.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern "C" int rbd_mirror_site_name_set(rados_t cluster, const char *name) {
|
||||
librados::Rados rados;
|
||||
librados::Rados::from_rados_t(cluster, rados);
|
||||
return librbd::api::Mirror<>::site_name_set(rados, name);
|
||||
}
|
||||
|
||||
extern "C" int rbd_mirror_mode_get(rados_ioctx_t p,
|
||||
rbd_mirror_mode_t *mirror_mode) {
|
||||
librados::IoCtx io_ctx;
|
||||
@ -2732,6 +2781,37 @@ extern "C" int rbd_mirror_mode_set(rados_ioctx_t p,
|
||||
return librbd::api::Mirror<>::mode_set(io_ctx, mirror_mode);
|
||||
}
|
||||
|
||||
extern "C" int rbd_mirror_peer_bootstrap_create(rados_ioctx_t p, char *token,
|
||||
size_t *max_len) {
|
||||
librados::IoCtx io_ctx;
|
||||
librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
|
||||
|
||||
std::string token_str;
|
||||
int r = librbd::api::Mirror<>::peer_bootstrap_create(io_ctx, &token_str);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
auto total_len = token_str.size() + 1;
|
||||
if (*max_len < total_len) {
|
||||
*max_len = total_len;
|
||||
return -ERANGE;
|
||||
}
|
||||
*max_len = total_len;
|
||||
|
||||
strcpy(token, token_str.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern "C" int rbd_mirror_peer_bootstrap_import(
|
||||
rados_ioctx_t p, rbd_mirror_peer_direction_t direction,
|
||||
const char *token) {
|
||||
librados::IoCtx io_ctx;
|
||||
librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
|
||||
|
||||
return librbd::api::Mirror<>::peer_bootstrap_import(io_ctx, direction, token);
|
||||
}
|
||||
|
||||
extern "C" int rbd_mirror_peer_add(rados_ioctx_t p, char *uuid,
|
||||
size_t uuid_max_length,
|
||||
const char *cluster_name,
|
||||
|
@ -19,6 +19,7 @@
|
||||
#include <boost/spirit/include/phoenix.hpp>
|
||||
#include <boost/fusion/adapted/struct/adapt_struct.hpp>
|
||||
#include <boost/fusion/include/adapt_struct.hpp>
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
|
||||
#include "MonCap.h"
|
||||
#include "include/stringify.h"
|
||||
@ -297,7 +298,7 @@ void MonCapGrant::expand_profile_mon(const EntityName& name) const
|
||||
profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
|
||||
profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
|
||||
}
|
||||
if (profile == "rbd" || profile == "rbd-mirror") {
|
||||
if (boost::starts_with(profile, "rbd")) {
|
||||
profile_grants.push_back(MonCapGrant("mon", MON_CAP_R));
|
||||
profile_grants.push_back(MonCapGrant("osd", MON_CAP_R));
|
||||
profile_grants.push_back(MonCapGrant("pg", MON_CAP_R));
|
||||
@ -314,6 +315,14 @@ void MonCapGrant::expand_profile_mon(const EntityName& name) const
|
||||
StringConstraint constraint(StringConstraint::MATCH_TYPE_PREFIX,
|
||||
"rbd/mirror/");
|
||||
profile_grants.push_back(MonCapGrant("config-key get", "key", constraint));
|
||||
} else if (profile == "rbd-mirror-peer") {
|
||||
StringConstraint constraint(StringConstraint::MATCH_TYPE_REGEX,
|
||||
"rbd/mirror/[^/]+");
|
||||
profile_grants.push_back(MonCapGrant("config-key get", "key", constraint));
|
||||
|
||||
constraint = StringConstraint(StringConstraint::MATCH_TYPE_PREFIX,
|
||||
"rbd/mirror/peer/");
|
||||
profile_grants.push_back(MonCapGrant("config-key set", "key", constraint));
|
||||
}
|
||||
|
||||
if (profile == "role-definer") {
|
||||
|
@ -97,6 +97,7 @@ cdef extern from "rbd/librbd.h" nogil:
|
||||
RBD_MAX_BLOCK_NAME_SIZE
|
||||
RBD_MAX_IMAGE_NAME_SIZE
|
||||
|
||||
ctypedef void* rados_t
|
||||
ctypedef void* rados_ioctx_t
|
||||
ctypedef void* rbd_image_t
|
||||
ctypedef void* rbd_image_options_t
|
||||
@ -153,6 +154,11 @@ cdef extern from "rbd/librbd.h" nogil:
|
||||
_RBD_MIRROR_MODE_IMAGE "RBD_MIRROR_MODE_IMAGE"
|
||||
_RBD_MIRROR_MODE_POOL "RBD_MIRROR_MODE_POOL"
|
||||
|
||||
ctypedef enum rbd_mirror_peer_direction_t:
|
||||
_RBD_MIRROR_PEER_DIRECTION_RX "RBD_MIRROR_PEER_DIRECTION_RX"
|
||||
_RBD_MIRROR_PEER_DIRECTION_TX "RBD_MIRROR_PEER_DIRECTION_TX"
|
||||
_RBD_MIRROR_PEER_DIRECTION_RX_TX "RBD_MIRROR_PEER_DIRECTION_RX_TX"
|
||||
|
||||
ctypedef struct rbd_mirror_peer_t:
|
||||
char *uuid
|
||||
char *cluster_name
|
||||
@ -340,8 +346,18 @@ cdef extern from "rbd/librbd.h" nogil:
|
||||
size_t status_size)
|
||||
void rbd_migration_status_cleanup(rbd_image_migration_status_t *status)
|
||||
|
||||
int rbd_mirror_site_name_get(rados_t cluster, char *name, size_t *max_len)
|
||||
int rbd_mirror_site_name_set(rados_t cluster, const char *name)
|
||||
|
||||
int rbd_mirror_mode_get(rados_ioctx_t io, rbd_mirror_mode_t *mirror_mode)
|
||||
int rbd_mirror_mode_set(rados_ioctx_t io, rbd_mirror_mode_t mirror_mode)
|
||||
|
||||
int rbd_mirror_peer_bootstrap_create(rados_ioctx_t io_ctx, char *token,
|
||||
size_t *max_len)
|
||||
int rbd_mirror_peer_bootstrap_import(
|
||||
rados_ioctx_t io_ctx, rbd_mirror_peer_direction_t direction,
|
||||
const char *token)
|
||||
|
||||
int rbd_mirror_peer_add(rados_ioctx_t io, char *uuid,
|
||||
size_t uuid_max_length, const char *cluster_name,
|
||||
const char *client_name)
|
||||
@ -648,6 +664,10 @@ RBD_MIRROR_MODE_DISABLED = _RBD_MIRROR_MODE_DISABLED
|
||||
RBD_MIRROR_MODE_IMAGE = _RBD_MIRROR_MODE_IMAGE
|
||||
RBD_MIRROR_MODE_POOL = _RBD_MIRROR_MODE_POOL
|
||||
|
||||
RBD_MIRROR_PEER_DIRECTION_RX = _RBD_MIRROR_PEER_DIRECTION_RX
|
||||
RBD_MIRROR_PEER_DIRECTION_TX = _RBD_MIRROR_PEER_DIRECTION_TX
|
||||
RBD_MIRROR_PEER_DIRECTION_RX_TX = _RBD_MIRROR_PEER_DIRECTION_RX_TX
|
||||
|
||||
RBD_MIRROR_IMAGE_DISABLING = _RBD_MIRROR_IMAGE_DISABLING
|
||||
RBD_MIRROR_IMAGE_ENABLED = _RBD_MIRROR_IMAGE_ENABLED
|
||||
RBD_MIRROR_IMAGE_DISABLED = _RBD_MIRROR_IMAGE_DISABLED
|
||||
@ -892,6 +912,9 @@ cdef make_ex(ret, msg, exception_map=errno_to_exception):
|
||||
return OSError(msg, errno=ret)
|
||||
|
||||
|
||||
cdef rados_t convert_rados(rados.Rados rados) except? NULL:
|
||||
return <rados_t>rados.cluster
|
||||
|
||||
cdef rados_ioctx_t convert_ioctx(rados.Ioctx ioctx) except? NULL:
|
||||
return <rados_ioctx_t>ioctx.io
|
||||
|
||||
@ -1684,6 +1707,50 @@ class RBD(object):
|
||||
|
||||
return status
|
||||
|
||||
def mirror_site_name_get(self, rados):
|
||||
"""
|
||||
Get the local cluster's friendly site name
|
||||
|
||||
:param rados: cluster connection
|
||||
:type rados: :class: rados.Rados
|
||||
:returns: str - local site name
|
||||
"""
|
||||
cdef:
|
||||
rados_t _rados = convert_rados(rados)
|
||||
char *_site_name = NULL
|
||||
size_t _max_size = 512
|
||||
try:
|
||||
while True:
|
||||
_site_name = <char *>realloc_chk(_site_name, _max_size)
|
||||
with nogil:
|
||||
ret = rbd_mirror_site_name_get(_rados, _site_name,
|
||||
&_max_size)
|
||||
if ret >= 0:
|
||||
break
|
||||
elif ret != -errno.ERANGE:
|
||||
raise make_ex(ret, 'error getting site name')
|
||||
return decode_cstr(_site_name)
|
||||
finally:
|
||||
free(_site_name)
|
||||
|
||||
def mirror_site_name_set(self, rados, site_name):
|
||||
"""
|
||||
Set the local cluster's friendly site name
|
||||
|
||||
:param rados: cluster connection
|
||||
:type rados: :class: rados.Rados
|
||||
:param site_name: friendly site name
|
||||
:type str:
|
||||
"""
|
||||
site_name = cstr(site_name, 'site_name')
|
||||
cdef:
|
||||
rados_t _rados = convert_rados(rados)
|
||||
char *_site_name = site_name
|
||||
with nogil:
|
||||
ret = rbd_mirror_site_name_set(_rados, _site_name)
|
||||
if ret != 0:
|
||||
raise make_ex(ret, 'error setting mirror site name')
|
||||
|
||||
def mirror_mode_get(self, ioctx):
|
||||
"""
|
||||
Get pool mirror mode.
|
||||
@ -1718,6 +1785,55 @@ class RBD(object):
|
||||
if ret != 0:
|
||||
raise make_ex(ret, 'error setting mirror mode')
|
||||
|
||||
def mirror_peer_bootstrap_create(self, ioctx):
|
||||
"""
|
||||
Creates a new RBD mirroring bootstrap token for an
|
||||
external cluster.
|
||||
|
||||
:param ioctx: determines which RADOS pool is written
|
||||
:type ioctx: :class:`rados.Ioctx`
|
||||
:returns: str - bootstrap token
|
||||
"""
|
||||
cdef:
|
||||
rados_ioctx_t _ioctx = convert_ioctx(ioctx)
|
||||
char *_token = NULL
|
||||
size_t _max_size = 512
|
||||
try:
|
||||
while True:
|
||||
_token = <char *>realloc_chk(_token, _max_size)
|
||||
with nogil:
|
||||
ret = rbd_mirror_peer_bootstrap_create(_ioctx, _token,
|
||||
&_max_size)
|
||||
if ret >= 0:
|
||||
break
|
||||
elif ret != -errno.ERANGE:
|
||||
raise make_ex(ret, 'error creating bootstrap token')
|
||||
return decode_cstr(_token)
|
||||
finally:
|
||||
free(_token)
|
||||
|
||||
def mirror_peer_bootstrap_import(self, ioctx, direction, token):
|
||||
"""
|
||||
Import a bootstrap token from an external cluster to
|
||||
auto-configure the mirror peer.
|
||||
|
||||
:param ioctx: determines which RADOS pool is written
|
||||
:type ioctx: :class:`rados.Ioctx`
|
||||
:param direction: mirror peer direction
|
||||
:type direction: int
|
||||
:param token: bootstrap token
|
||||
:type token: str
|
||||
"""
|
||||
token = cstr(token, 'token')
|
||||
cdef:
|
||||
rados_ioctx_t _ioctx = convert_ioctx(ioctx)
|
||||
rbd_mirror_peer_direction_t _direction = direction
|
||||
char *_token = token
|
||||
with nogil:
|
||||
ret = rbd_mirror_peer_bootstrap_import(_ioctx, _direction, _token)
|
||||
if ret != 0:
|
||||
raise make_ex(ret, 'error importing bootstrap token')
|
||||
|
||||
def mirror_peer_add(self, ioctx, cluster_name, client_name):
|
||||
"""
|
||||
Add mirror peer.
|
||||
|
@ -96,6 +96,10 @@
|
||||
mirror pool info Show information about the pool mirroring
|
||||
configuration.
|
||||
mirror pool peer add Add a mirroring peer to a pool.
|
||||
mirror pool peer bootstrap create Create a peer bootstrap token to import
|
||||
in a remote cluster
|
||||
mirror pool peer bootstrap import Import a peer bootstrap token created
|
||||
from a remote cluster
|
||||
mirror pool peer remove Remove a mirroring peer from a pool.
|
||||
mirror pool peer set Update mirroring peer settings.
|
||||
mirror pool promote Promote all non-primary images in the
|
||||
@ -1631,6 +1635,7 @@
|
||||
|
||||
rbd help mirror pool enable
|
||||
usage: rbd mirror pool enable [--pool <pool>] [--namespace <namespace>]
|
||||
[--site-name <site-name>]
|
||||
<pool-spec> <mode>
|
||||
|
||||
Enable RBD mirroring by default within a pool.
|
||||
@ -1643,6 +1648,7 @@
|
||||
Optional arguments
|
||||
-p [ --pool ] arg pool name
|
||||
--namespace arg namespace name
|
||||
--site-name arg local site name
|
||||
|
||||
rbd help mirror pool info
|
||||
usage: rbd mirror pool info [--pool <pool>] [--namespace <namespace>]
|
||||
@ -1684,6 +1690,42 @@
|
||||
--remote-mon-host arg remote mon host(s)
|
||||
--remote-key-file arg path to file containing remote key
|
||||
|
||||
rbd help mirror pool peer bootstrap create
|
||||
usage: rbd mirror pool peer bootstrap create
|
||||
[--pool <pool>]
|
||||
[--site-name <site-name>]
|
||||
<pool-name>
|
||||
|
||||
Create a peer bootstrap token to import in a remote cluster
|
||||
|
||||
Positional arguments
|
||||
<pool-name> pool name
|
||||
|
||||
Optional arguments
|
||||
-p [ --pool ] arg pool name
|
||||
--site-name arg local site name
|
||||
|
||||
rbd help mirror pool peer bootstrap import
|
||||
usage: rbd mirror pool peer bootstrap import
|
||||
[--pool <pool>]
|
||||
[--site-name <site-name>]
|
||||
[--token-path <token-path>]
|
||||
[--direction <direction>]
|
||||
<pool-name> <token-path>
|
||||
|
||||
Import a peer bootstrap token created from a remote cluster
|
||||
|
||||
Positional arguments
|
||||
<pool-name> pool name
|
||||
<token-path> bootstrap token file (or '-' for stdin)
|
||||
|
||||
Optional arguments
|
||||
-p [ --pool ] arg pool name
|
||||
--site-name arg local site name
|
||||
--token-path arg bootstrap token file (or '-' for stdin)
|
||||
--direction arg mirroring direction (rx-only, rx-tx)
|
||||
[default: rx-tx]
|
||||
|
||||
rbd help mirror pool peer remove
|
||||
usage: rbd mirror pool peer remove [--pool <pool>]
|
||||
<pool-name> <uuid>
|
||||
|
@ -991,6 +991,19 @@ Rados::~Rados() {
|
||||
shutdown();
|
||||
}
|
||||
|
||||
void Rados::from_rados_t(rados_t p, Rados &rados) {
|
||||
if (rados.client != nullptr) {
|
||||
reinterpret_cast<TestRadosClient*>(rados.client)->put();
|
||||
rados.client = nullptr;
|
||||
}
|
||||
|
||||
auto impl = reinterpret_cast<TestRadosClient*>(p);
|
||||
if (impl) {
|
||||
impl->get();
|
||||
rados.client = reinterpret_cast<RadosClient*>(impl);
|
||||
}
|
||||
}
|
||||
|
||||
AioCompletion *Rados::aio_create_completion(void *cb_arg,
|
||||
callback_t cb_complete,
|
||||
callback_t cb_safe) {
|
||||
|
@ -950,3 +950,43 @@ TEST_F(TestMirroring, AioGetStatus) {
|
||||
ASSERT_EQ(0, status.last_update);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestMirroring, SiteName) {
|
||||
REQUIRE(!is_librados_test_stub(_rados));
|
||||
|
||||
const std::string expected_site_name("us-east-1a");
|
||||
ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, expected_site_name));
|
||||
|
||||
std::string site_name;
|
||||
ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name));
|
||||
ASSERT_EQ(expected_site_name, site_name);
|
||||
|
||||
ASSERT_EQ(0, m_rbd.mirror_site_name_set(_rados, ""));
|
||||
|
||||
std::string fsid;
|
||||
ASSERT_EQ(0, _rados.cluster_fsid(&fsid));
|
||||
ASSERT_EQ(0, m_rbd.mirror_site_name_get(_rados, &site_name));
|
||||
ASSERT_EQ(fsid, site_name);
|
||||
}
|
||||
|
||||
TEST_F(TestMirroring, Bootstrap) {
|
||||
REQUIRE(!is_librados_test_stub(_rados));
|
||||
|
||||
std::string token_b64;
|
||||
ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
|
||||
ASSERT_EQ(-EINVAL, m_rbd.mirror_peer_bootstrap_create(m_ioctx, &token_b64));
|
||||
|
||||
ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
|
||||
ASSERT_EQ(0, m_rbd.mirror_peer_bootstrap_create(m_ioctx, &token_b64));
|
||||
|
||||
bufferlist token_b64_bl;
|
||||
token_b64_bl.append(token_b64);
|
||||
|
||||
bufferlist token_bl;
|
||||
token_bl.decode_base64(token_b64_bl);
|
||||
|
||||
// cannot import token into same cluster
|
||||
ASSERT_EQ(-EINVAL,
|
||||
m_rbd.mirror_peer_bootstrap_import(
|
||||
m_ioctx, RBD_MIRROR_PEER_DIRECTION_RX, token_b64));
|
||||
}
|
||||
|
@ -1,6 +1,8 @@
|
||||
# vim: expandtab smarttab shiftwidth=4 softtabstop=4
|
||||
import base64
|
||||
import errno
|
||||
import functools
|
||||
import json
|
||||
import socket
|
||||
import os
|
||||
import time
|
||||
@ -28,7 +30,8 @@ from rbd import (RBD, Group, Image, ImageNotFound, InvalidArgument, ImageExists,
|
||||
RBD_IMAGE_MIGRATION_STATE_PREPARED, RBD_CONFIG_SOURCE_CONFIG,
|
||||
RBD_CONFIG_SOURCE_POOL, RBD_CONFIG_SOURCE_IMAGE,
|
||||
RBD_MIRROR_PEER_ATTRIBUTE_NAME_MON_HOST,
|
||||
RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY)
|
||||
RBD_MIRROR_PEER_ATTRIBUTE_NAME_KEY,
|
||||
RBD_MIRROR_PEER_DIRECTION_RX)
|
||||
|
||||
rados = None
|
||||
ioctx = None
|
||||
@ -1789,6 +1792,30 @@ class TestMirroring(object):
|
||||
remove_image()
|
||||
self.rbd.mirror_mode_set(ioctx, self.initial_mirror_mode)
|
||||
|
||||
def test_site_name(self):
|
||||
site_name = "us-west-1"
|
||||
self.rbd.mirror_site_name_set(rados, site_name)
|
||||
eq(site_name, self.rbd.mirror_site_name_get(rados))
|
||||
self.rbd.mirror_site_name_set(rados, "")
|
||||
eq(rados.get_fsid(), self.rbd.mirror_site_name_get(rados))
|
||||
|
||||
def test_mirror_peer_bootstrap(self):
|
||||
eq([], list(self.rbd.mirror_peer_list(ioctx)))
|
||||
|
||||
self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_DISABLED)
|
||||
assert_raises(InvalidArgument, self.rbd.mirror_peer_bootstrap_create,
|
||||
ioctx);
|
||||
|
||||
self.rbd.mirror_mode_set(ioctx, RBD_MIRROR_MODE_POOL)
|
||||
token_b64 = self.rbd.mirror_peer_bootstrap_create(ioctx)
|
||||
token = base64.b64decode(token_b64)
|
||||
token_dict = json.loads(token)
|
||||
eq(sorted(['fsid', 'client_id', 'key', 'mon_host']),
|
||||
sorted(list(token_dict.keys())))
|
||||
|
||||
# requires different cluster
|
||||
assert_raises(InvalidArgument, self.rbd.mirror_peer_bootstrap_import,
|
||||
ioctx, RBD_MIRROR_PEER_DIRECTION_RX, token_b64)
|
||||
|
||||
def test_mirror_peer(self):
|
||||
eq([], list(self.rbd.mirror_peer_list(ioctx)))
|
||||
|
@ -39,9 +39,46 @@ namespace at = argument_types;
|
||||
namespace po = boost::program_options;
|
||||
|
||||
static const std::string ALL_NAME("all");
|
||||
static const std::string SITE_NAME("site-name");
|
||||
|
||||
namespace {
|
||||
|
||||
void add_site_name_optional(po::options_description *options) {
|
||||
options->add_options()
|
||||
(SITE_NAME.c_str(), po::value<std::string>(), "local site name");
|
||||
}
|
||||
|
||||
int set_site_name(librados::Rados& rados, const std::string& site_name) {
|
||||
librbd::RBD rbd;
|
||||
int r = rbd.mirror_site_name_set(rados, site_name);
|
||||
if (r == -EOPNOTSUPP) {
|
||||
std::cerr << "rbd: cluster does not support site names" << std::endl;
|
||||
return r;
|
||||
} else if (r < 0) {
|
||||
std::cerr << "rbd: failed to set site name" << cpp_strerror(r)
|
||||
<< std::endl;
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct MirrorPeerDirection {};
|
||||
|
||||
void validate(boost::any& v, const std::vector<std::string>& values,
|
||||
MirrorPeerDirection *target_type, int) {
|
||||
po::validators::check_first_occurrence(v);
|
||||
const std::string &s = po::validators::get_single_string(values);
|
||||
|
||||
if (s == "rx-only") {
|
||||
v = boost::any(RBD_MIRROR_PEER_DIRECTION_RX);
|
||||
} else if (s == "rx-tx") {
|
||||
v = boost::any(RBD_MIRROR_PEER_DIRECTION_RX_TX);
|
||||
} else {
|
||||
throw po::validation_error(po::validation_error::invalid_option_value);
|
||||
}
|
||||
}
|
||||
|
||||
int validate_mirroring_enabled(librados::IoCtx& io_ctx) {
|
||||
librbd::RBD rbd;
|
||||
rbd_mirror_mode_t mirror_mode;
|
||||
@ -653,6 +690,154 @@ private:
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void get_peer_bootstrap_create_arguments(po::options_description *positional,
|
||||
po::options_description *options) {
|
||||
at::add_pool_options(positional, options, false);
|
||||
options->add_options()
|
||||
(SITE_NAME.c_str(), po::value<std::string>(), "local site name");
|
||||
}
|
||||
|
||||
int execute_peer_bootstrap_create(
|
||||
const po::variables_map &vm,
|
||||
const std::vector<std::string> &ceph_global_init_args) {
|
||||
std::string pool_name;
|
||||
size_t arg_index = 0;
|
||||
int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
|
||||
nullptr, &arg_index);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
librados::Rados rados;
|
||||
librados::IoCtx io_ctx;
|
||||
r = utils::init(pool_name, "", &rados, &io_ctx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
r = validate_mirroring_enabled(io_ctx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
if (vm.count(SITE_NAME)) {
|
||||
r = set_site_name(rados, vm[SITE_NAME].as<std::string>());
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
librbd::RBD rbd;
|
||||
std::string token;
|
||||
r = rbd.mirror_peer_bootstrap_create(io_ctx, &token);
|
||||
if (r == -EEXIST) {
|
||||
std::cerr << "rbd: mismatch with pre-existing RBD mirroring peer user caps"
|
||||
<< std::endl;
|
||||
} else if (r < 0) {
|
||||
std::cerr << "rbd: failed to create mirroring bootstrap token: "
|
||||
<< cpp_strerror(r) << std::endl;
|
||||
return r;
|
||||
}
|
||||
|
||||
std::cout << token << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void get_peer_bootstrap_import_arguments(po::options_description *positional,
|
||||
po::options_description *options) {
|
||||
at::add_pool_options(positional, options, false);
|
||||
options->add_options()
|
||||
(SITE_NAME.c_str(), po::value<std::string>(), "local site name");
|
||||
positional->add_options()
|
||||
("token-path", po::value<std::string>(),
|
||||
"bootstrap token file (or '-' for stdin)");
|
||||
options->add_options()
|
||||
("token-path", po::value<std::string>(),
|
||||
"bootstrap token file (or '-' for stdin)")
|
||||
("direction", po::value<MirrorPeerDirection>(),
|
||||
"mirroring direction (rx-only, rx-tx)\n"
|
||||
"[default: rx-tx]");
|
||||
}
|
||||
|
||||
int execute_peer_bootstrap_import(
|
||||
const po::variables_map &vm,
|
||||
const std::vector<std::string> &ceph_global_init_args) {
|
||||
std::string pool_name;
|
||||
size_t arg_index = 0;
|
||||
int r = utils::get_pool_and_namespace_names(vm, true, true, &pool_name,
|
||||
nullptr, &arg_index);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string token_path;
|
||||
if (vm.count("token-path")) {
|
||||
token_path = vm["token-path"].as<std::string>();
|
||||
} else {
|
||||
token_path = utils::get_positional_argument(vm, arg_index++);
|
||||
}
|
||||
|
||||
if (token_path.empty()) {
|
||||
std::cerr << "rbd: token path was not specified" << std::endl;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rbd_mirror_peer_direction_t mirror_peer_direction =
|
||||
RBD_MIRROR_PEER_DIRECTION_RX_TX;
|
||||
if (vm.count("direction")) {
|
||||
mirror_peer_direction = vm["direction"].as<rbd_mirror_peer_direction_t>();
|
||||
}
|
||||
|
||||
int fd = STDIN_FILENO;
|
||||
if (token_path != "-") {
|
||||
fd = open(token_path.c_str(), O_RDONLY);
|
||||
if (fd < 0) {
|
||||
r = -errno;
|
||||
std::cerr << "rbd: error opening " << token_path << std::endl;
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
char token[1024];
|
||||
memset(token, 0, sizeof(token));
|
||||
r = safe_read(fd, token, sizeof(token) - 1);
|
||||
if (fd != STDIN_FILENO) {
|
||||
VOID_TEMP_FAILURE_RETRY(close(fd));
|
||||
}
|
||||
|
||||
if (r < 0) {
|
||||
std::cerr << "rbd: error reading token file: " << cpp_strerror(r)
|
||||
<< std::endl;
|
||||
return r;
|
||||
}
|
||||
|
||||
librados::Rados rados;
|
||||
librados::IoCtx io_ctx;
|
||||
r = utils::init(pool_name, "", &rados, &io_ctx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
if (vm.count(SITE_NAME)) {
|
||||
r = set_site_name(rados, vm[SITE_NAME].as<std::string>());
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
librbd::RBD rbd;
|
||||
r = rbd.mirror_peer_bootstrap_import(io_ctx, mirror_peer_direction, token);
|
||||
if (r == -ENOSYS) {
|
||||
std::cerr << "rbd: mirroring is not enabled on remote peer" << std::endl;
|
||||
return r;
|
||||
} else if (r < 0) {
|
||||
std::cerr << "rbd: failed to import peer bootstrap token" << std::endl;
|
||||
return r;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void get_peer_add_arguments(po::options_description *positional,
|
||||
po::options_description *options) {
|
||||
at::add_pool_options(positional, options, false);
|
||||
@ -868,23 +1053,15 @@ void get_enable_arguments(po::options_description *positional,
|
||||
at::add_pool_options(positional, options, true);
|
||||
positional->add_options()
|
||||
("mode", "mirror mode [image or pool]");
|
||||
add_site_name_optional(options);
|
||||
}
|
||||
|
||||
int execute_enable_disable(const std::string &pool_name,
|
||||
const std::string &namespace_name,
|
||||
int execute_enable_disable(librados::IoCtx& io_ctx,
|
||||
rbd_mirror_mode_t next_mirror_mode,
|
||||
const std::string &mode) {
|
||||
librados::Rados rados;
|
||||
librados::IoCtx io_ctx;
|
||||
rbd_mirror_mode_t current_mirror_mode;
|
||||
|
||||
int r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
const std::string &mode, bool ignore_no_update) {
|
||||
librbd::RBD rbd;
|
||||
r = rbd.mirror_mode_get(io_ctx, ¤t_mirror_mode);
|
||||
rbd_mirror_mode_t current_mirror_mode;
|
||||
int r = rbd.mirror_mode_get(io_ctx, ¤t_mirror_mode);
|
||||
if (r < 0) {
|
||||
std::cerr << "rbd: failed to retrieve mirror mode: "
|
||||
<< cpp_strerror(r) << std::endl;
|
||||
@ -892,11 +1069,13 @@ int execute_enable_disable(const std::string &pool_name,
|
||||
}
|
||||
|
||||
if (current_mirror_mode == next_mirror_mode) {
|
||||
if (mode == "disabled") {
|
||||
std::cout << "mirroring is already " << mode << std::endl;
|
||||
} else {
|
||||
std::cout << "mirroring is already configured for "
|
||||
<< mode << " mode" << std::endl;
|
||||
if (!ignore_no_update) {
|
||||
if (mode == "disabled") {
|
||||
std::cout << "rbd: mirroring is already " << mode << std::endl;
|
||||
} else {
|
||||
std::cout << "rbd: mirroring is already configured for "
|
||||
<< mode << " mode" << std::endl;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
} else if (next_mirror_mode == RBD_MIRROR_MODE_IMAGE &&
|
||||
@ -927,8 +1106,16 @@ int execute_disable(const po::variables_map &vm,
|
||||
return r;
|
||||
}
|
||||
|
||||
return execute_enable_disable(pool_name, namespace_name,
|
||||
RBD_MIRROR_MODE_DISABLED, "disabled");
|
||||
librados::Rados rados;
|
||||
librados::IoCtx io_ctx;
|
||||
|
||||
r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
return execute_enable_disable(io_ctx, RBD_MIRROR_MODE_DISABLED, "disabled",
|
||||
false);
|
||||
}
|
||||
|
||||
int execute_enable(const po::variables_map &vm,
|
||||
@ -953,7 +1140,30 @@ int execute_enable(const po::variables_map &vm,
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return execute_enable_disable(pool_name, namespace_name, mirror_mode, mode);
|
||||
librados::Rados rados;
|
||||
librados::IoCtx io_ctx;
|
||||
|
||||
r = utils::init(pool_name, namespace_name, &rados, &io_ctx);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
|
||||
bool updated = false;
|
||||
if (vm.count(SITE_NAME)) {
|
||||
librbd::RBD rbd;
|
||||
|
||||
auto site_name = vm[SITE_NAME].as<std::string>();
|
||||
std::string original_site_name;
|
||||
r = rbd.mirror_site_name_get(rados, &original_site_name);
|
||||
updated = (r >= 0 && site_name != original_site_name);
|
||||
|
||||
r = set_site_name(rados, site_name);
|
||||
if (r < 0) {
|
||||
return r;
|
||||
}
|
||||
}
|
||||
|
||||
return execute_enable_disable(io_ctx, mirror_mode, mode, updated);
|
||||
}
|
||||
|
||||
void get_info_arguments(po::options_description *positional,
|
||||
@ -995,6 +1205,12 @@ int execute_info(const po::variables_map &vm,
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string site_name;
|
||||
r = rbd.mirror_site_name_get(rados, &site_name);
|
||||
if (r < 0 && r != -EOPNOTSUPP) {
|
||||
return r;
|
||||
}
|
||||
|
||||
std::vector<librbd::mirror_peer_t> mirror_peers;
|
||||
if (namespace_name.empty()) {
|
||||
r = rbd.mirror_peer_list(io_ctx, &mirror_peers);
|
||||
@ -1027,6 +1243,12 @@ int execute_info(const po::variables_map &vm,
|
||||
}
|
||||
|
||||
if (mirror_mode != RBD_MIRROR_MODE_DISABLED && namespace_name.empty()) {
|
||||
if (formatter != nullptr) {
|
||||
formatter->dump_string("site_name", site_name);
|
||||
} else {
|
||||
std::cout << "Site Name: " << site_name << std::endl;
|
||||
}
|
||||
|
||||
r = format_mirror_peers(io_ctx, formatter, mirror_peers,
|
||||
vm[ALL_NAME].as<bool>());
|
||||
if (r < 0) {
|
||||
@ -1262,6 +1484,15 @@ int execute_demote(const po::variables_map &vm,
|
||||
return r;
|
||||
}
|
||||
|
||||
Shell::Action action_bootstrap_create(
|
||||
{"mirror", "pool", "peer", "bootstrap", "create"}, {},
|
||||
"Create a peer bootstrap token to import in a remote cluster", "",
|
||||
&get_peer_bootstrap_create_arguments, &execute_peer_bootstrap_create);
|
||||
Shell::Action action_bootstreap_import(
|
||||
{"mirror", "pool", "peer", "bootstrap", "import"}, {},
|
||||
"Import a peer bootstrap token created from a remote cluster", "",
|
||||
&get_peer_bootstrap_import_arguments, &execute_peer_bootstrap_import);
|
||||
|
||||
Shell::Action action_add(
|
||||
{"mirror", "pool", "peer", "add"}, {},
|
||||
"Add a mirroring peer to a pool.", "",
|
||||
|
Loading…
Reference in New Issue
Block a user