mirror of
https://github.com/ceph/ceph
synced 2025-01-01 08:32:24 +00:00
rados tool: integrate rados_sync with rados tool
* integrate rados_sync with rados_tool * Improve rados tool usage a bit * Rename test_rados_sync.sh to test_rados_tool.sh Signed-off-by: Colin McCabe <colin.mccabe@dreamhost.com>
This commit is contained in:
Colin Patrick McCabe
parent
15756550fb
commit
3425a8e503
@ -260,14 +260,10 @@ librbd_a_SOURCES = ${librbd_SOURCES}
|
||||
librbd_a_CFLAGS = ${AM_CFLAGS}
|
||||
librbd_a_CXXFLAGS = ${AM_CXXFLAGS}
|
||||
|
||||
rados_SOURCES = rados.cc
|
||||
rados_SOURCES = rados.cc rados_sync.cc
|
||||
rados_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS)
|
||||
bin_PROGRAMS += rados
|
||||
|
||||
rados_sync_SOURCES = rados_sync.cc
|
||||
rados_sync_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS)
|
||||
bin_PROGRAMS += rados_sync
|
||||
|
||||
testrados_SOURCES = testrados.c
|
||||
testrados_LDADD = librados.la -lpthread -lm $(CRYPTO_LIBS) $(EXTRALIBS)
|
||||
testradospp_SOURCES = testradospp.cc
|
||||
|
||||
125
src/rados.cc
125
src/rados.cc
@ -33,58 +33,63 @@ using namespace librados;
|
||||
#include <errno.h>
|
||||
#include <dirent.h>
|
||||
|
||||
int rados_tool_sync(const std::map < std::string, std::string > &opts,
|
||||
std::vector<const char*> &args);
|
||||
|
||||
void usage()
|
||||
{
|
||||
cerr << "usage: rados [options] [commands]" << std::endl;
|
||||
/* cerr << "If no commands are specified, enter interactive mode.\n";
|
||||
cerr << "Commands:" << std::endl;
|
||||
cerr << " stop -- cleanly shut down file system" << std::endl
|
||||
<< " (osd|pg|mds) stat -- get monitor subsystem status" << std::endl
|
||||
<< " ..." << std::endl;
|
||||
*/
|
||||
cerr << "Commands:\n";
|
||||
cerr << " lspools list pools\n";
|
||||
cerr << " df show per-pool and total usage\n\n";
|
||||
|
||||
cerr << "Pool commands:\n";
|
||||
cerr << " get <obj-name> [outfile] fetch object\n";
|
||||
cerr << " put <obj-name> [infile] write object\n";
|
||||
cerr << " create <obj-name> create object\n";
|
||||
cerr << " rm <obj-name> remove object\n";
|
||||
cerr << " listxattr <obj-name>\n";
|
||||
cerr << " getxattr <obj-name> attr\n";
|
||||
cerr << " setxattr <obj-name> attr val\n";
|
||||
cerr << " rmxattr <obj-name> attr\n";
|
||||
cerr << " stat objname stat the named object\n";
|
||||
cerr << " ls list objects in pool\n\n";
|
||||
cerr << " chown 123 change the pool owner to auid 123\n";
|
||||
cerr << " mapext <obj-name>\n";
|
||||
cerr << " mkpool <pool-name> [123[ 4]] create pool <pool-name>'\n"
|
||||
<< " [with auid 123[and using crush rule 4]]\n";
|
||||
cerr << " rmpool <pool-name> remove pool <pool-name>'\n";
|
||||
cerr << " mkpool <pool-name> create the pool <pool-name>\n";
|
||||
cerr << " lssnap list snaps\n";
|
||||
cerr << " mksnap <snap-name> create snap <snap-name>\n";
|
||||
cerr << " rmsnap <snap-name> remove snap <snap-name>\n";
|
||||
cerr << " rollback <obj-name> <snap-name> roll back object to snap <snap-name>\n\n";
|
||||
cerr << " bench <seconds> write|seq|rand [-t concurrent_operations]\n";
|
||||
cerr << " default is 16 concurrent IOs and 4 MB op size\n\n";
|
||||
|
||||
cerr << "Options:\n";
|
||||
cerr << " -p pool\n";
|
||||
cerr << " --pool=pool\n";
|
||||
cerr << " select given pool by name\n";
|
||||
cerr << " -b op_size\n";
|
||||
cerr << " set the size of write ops for put or benchmarking";
|
||||
cerr << " -s name\n";
|
||||
cerr << " --snap name\n";
|
||||
cerr << " select given snap name for (read) IO\n";
|
||||
cerr << " -i infile\n";
|
||||
cerr << " -o outfile\n";
|
||||
cerr << " specify input or output file (for certain commands)\n";
|
||||
cerr << " --create-pool\n";
|
||||
cerr << " create the pool that was specified\n";
|
||||
exit(1);
|
||||
cerr << \
|
||||
"usage: rados [options] [commands]\n"
|
||||
"POOL COMMANDS\n"
|
||||
" lspools list pools\n"
|
||||
" mkpool <pool-name> [123[ 4]] create pool <pool-name>'\n"
|
||||
" [with auid 123[and using crush rule 4]]\n"
|
||||
" rmpool <pool-name> remove pool <pool-name>'\n"
|
||||
" mkpool <pool-name> create the pool <pool-name>\n"
|
||||
" df show per-pool and total usage\n"
|
||||
" ls list objects in pool\n\n"
|
||||
" chown 123 change the pool owner to auid 123\n"
|
||||
"OBJECT COMMANDS\n"
|
||||
" get <obj-name> [outfile] fetch object\n"
|
||||
" put <obj-name> [infile] write object\n"
|
||||
" create <obj-name> create object\n"
|
||||
" rm <obj-name> remove object\n"
|
||||
" listxattr <obj-name>\n"
|
||||
" getxattr <obj-name> attr\n"
|
||||
" setxattr <obj-name> attr val\n"
|
||||
" rmxattr <obj-name> attr\n"
|
||||
" stat objname stat the named object\n"
|
||||
" mapext <obj-name>\n"
|
||||
" lssnap list snaps\n"
|
||||
" mksnap <snap-name> create snap <snap-name>\n"
|
||||
" rmsnap <snap-name> remove snap <snap-name>\n"
|
||||
" rollback <obj-name> <snap-name> roll back object to snap <snap-name>\n\n"
|
||||
" bench <seconds> write|seq|rand [-t concurrent_operations]\n"
|
||||
" default is 16 concurrent IOs and 4 MB op size\n\n"
|
||||
"IMPORT AND EXPORT\n"
|
||||
" import [options] <local-directory> <rados-pool>\n"
|
||||
" Upload <local-directory> to <rados-pool>\n"
|
||||
" export [options] rados-pool> <local-directory>\n"
|
||||
" Download <rados-pool> to <local-directory>\n"
|
||||
" options:\n"
|
||||
" -f / --force Copy everything, even if it hasn't changed.\n"
|
||||
" -d / --delete-after After synchronizing, delete unreferenced\n"
|
||||
" files or objects from the target bucket\n"
|
||||
" or directory.\n"
|
||||
"GLOBAL OPTIONS:\n"
|
||||
" -p pool\n"
|
||||
" --pool=pool\n"
|
||||
" select given pool by name\n"
|
||||
" -b op_size\n"
|
||||
" set the size of write ops for put or benchmarking"
|
||||
" -s name\n"
|
||||
" --snap name\n"
|
||||
" select given snap name for (read) IO\n"
|
||||
" -i infile\n"
|
||||
" -o outfile\n"
|
||||
" specify input or output file (for certain commands)\n"
|
||||
" --create\n"
|
||||
" create the pool or directory that was specified\n";
|
||||
}
|
||||
|
||||
static int do_get(IoCtx& io_ctx, const char *objname, const char *outfile, bool check_stdio)
|
||||
@ -196,11 +201,6 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
|
||||
snapid = strtoll(i->second.c_str(), NULL, 10);
|
||||
}
|
||||
|
||||
if (nargs.empty()) {
|
||||
usage();
|
||||
return 1;
|
||||
}
|
||||
|
||||
// open rados
|
||||
Rados rados;
|
||||
ret = rados.init_with_config(&g_conf);
|
||||
@ -261,6 +261,8 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts,
|
||||
cout << "selected snap " << snapid << " '" << snapname << "'" << std::endl;
|
||||
}
|
||||
|
||||
assert(!nargs.empty());
|
||||
|
||||
// list pools?
|
||||
if (strcmp(nargs[0], "lspools") == 0) {
|
||||
list<string> vec;
|
||||
@ -666,6 +668,10 @@ int main(int argc, const char **argv)
|
||||
if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
|
||||
usage();
|
||||
exit(0);
|
||||
} else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) {
|
||||
opts["force"] = "true";
|
||||
} else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) {
|
||||
opts["delete-after"] = "true";
|
||||
} else if (ceph_argparse_flag(args, i, "-C", "--create", "--create-pool",
|
||||
(char*)NULL)) {
|
||||
opts["create"] = "true";
|
||||
@ -683,5 +689,12 @@ int main(int argc, const char **argv)
|
||||
}
|
||||
}
|
||||
|
||||
return rados_tool_common(opts, args);
|
||||
if (args.empty()) {
|
||||
cerr << "rados: you must give an action. Try --help" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
if ((strcmp(args[0], "import") == 0) || (strcmp(args[0], "export") == 0))
|
||||
return rados_tool_sync(opts, args);
|
||||
else
|
||||
return rados_tool_common(opts, args);
|
||||
}
|
||||
|
||||
@ -46,22 +46,6 @@ static const char ERR_PREFIX[] = "[ERROR] ";
|
||||
#define ENOATTR ENODATA
|
||||
#endif
|
||||
|
||||
static void usage()
|
||||
{
|
||||
cerr << "usage:\n\
|
||||
\n\
|
||||
Importing data from a local directory to a rados pool:\n\
|
||||
rados_sync [options] import <local-directory> <rados-pool>\n\
|
||||
\n\
|
||||
Exporting data from a rados pool to a local directory:\n\
|
||||
rados_sync [options] export <rados-pool> <local-directory>\n\
|
||||
\n\
|
||||
options:\n\
|
||||
-h or --help This help message\n\
|
||||
-c or --create Create destination pools or directories that don't exist\n\
|
||||
";
|
||||
}
|
||||
|
||||
static int xattr_test(const char *dir_name)
|
||||
{
|
||||
int ret;
|
||||
@ -961,41 +945,22 @@ static int do_import(IoCtx& io_ctx, const char *dir_name,
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, const char **argv)
|
||||
int rados_tool_sync(const std::map < std::string, std::string > &opts,
|
||||
std::vector<const char*> &args)
|
||||
{
|
||||
int ret;
|
||||
bool create = false;
|
||||
bool force = false;
|
||||
bool delete_after = false;
|
||||
vector<const char*> args;
|
||||
bool force = opts.count("force");
|
||||
bool delete_after = opts.count("delete-after");
|
||||
bool create = opts.count("create");
|
||||
std::string action, src, dst;
|
||||
argv_to_vec(argc, argv, args);
|
||||
env_to_vec(args);
|
||||
common_init(args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
|
||||
|
||||
std::vector<const char*>::iterator i;
|
||||
for (i = args.begin(); i != args.end(); ) {
|
||||
if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
|
||||
usage();
|
||||
exit(1);
|
||||
} else if (ceph_argparse_flag(args, i, "-C", "--create", (char*)NULL)) {
|
||||
create = true;
|
||||
} else if (ceph_argparse_flag(args, i, "-f", "--force", (char*)NULL)) {
|
||||
force = true;
|
||||
} else if (ceph_argparse_flag(args, i, "-d", "--delete-after", (char*)NULL)) {
|
||||
delete_after = true;
|
||||
} else {
|
||||
// begin positional arguments
|
||||
break;
|
||||
}
|
||||
}
|
||||
std::vector<const char*>::iterator i = args.begin();
|
||||
if ((i != args.end()) &&
|
||||
((strcmp(*i, "import") == 0) || (strcmp(*i, "export") == 0))) {
|
||||
action = *i;
|
||||
++i;
|
||||
}
|
||||
else {
|
||||
cerr << argv[0] << ": You must specify either 'import' or 'export'.\n";
|
||||
cerr << "rados" << ": You must specify either 'import' or 'export'.\n";
|
||||
cerr << "Use --help to show help.\n";
|
||||
exit(1);
|
||||
}
|
||||
@ -1004,7 +969,7 @@ int main(int argc, const char **argv)
|
||||
++i;
|
||||
}
|
||||
else {
|
||||
cerr << argv[0] << ": You must give a source.\n";
|
||||
cerr << "rados" << ": You must give a source.\n";
|
||||
cerr << "Use --help to show help.\n";
|
||||
exit(1);
|
||||
}
|
||||
@ -1013,7 +978,7 @@ int main(int argc, const char **argv)
|
||||
++i;
|
||||
}
|
||||
else {
|
||||
cerr << argv[0] << ": You must give a destination.\n";
|
||||
cerr << "rados" << ": You must give a destination.\n";
|
||||
cerr << "Use --help to show help.\n";
|
||||
exit(1);
|
||||
}
|
||||
@ -1021,11 +986,11 @@ int main(int argc, const char **argv)
|
||||
// open rados
|
||||
Rados rados;
|
||||
if (rados.init_with_config(&g_conf) < 0) {
|
||||
cerr << argv[0] << ": failed to initialize Rados!" << std::endl;
|
||||
cerr << "rados" << ": failed to initialize Rados!" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
if (rados.connect() < 0) {
|
||||
cerr << argv[0] << ": failed to connect to Rados cluster!" << std::endl;
|
||||
cerr << "rados" << ": failed to connect to Rados cluster!" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
IoCtx io_ctx;
|
||||
@ -1035,20 +1000,20 @@ int main(int argc, const char **argv)
|
||||
if (create) {
|
||||
ret = rados.pool_create(pool_name.c_str());
|
||||
if (ret) {
|
||||
cerr << argv[0] << ": pool_create failed with error " << ret
|
||||
cerr << "rados" << ": pool_create failed with error " << ret
|
||||
<< std::endl;
|
||||
exit(ret);
|
||||
}
|
||||
ret = rados.ioctx_create(pool_name.c_str(), io_ctx);
|
||||
}
|
||||
else {
|
||||
cerr << argv[0] << ": pool '" << pool_name << "' does not exist. Use "
|
||||
cerr << "rados" << ": pool '" << pool_name << "' does not exist. Use "
|
||||
<< "--create to try to create it." << std::endl;
|
||||
exit(ENOENT);
|
||||
}
|
||||
}
|
||||
if (ret < 0) {
|
||||
cerr << argv[0] << ": error opening pool " << pool_name << ": "
|
||||
cerr << "rados" << ": error opening pool " << pool_name << ": "
|
||||
<< cpp_strerror(ret) << std::endl;
|
||||
exit(ret);
|
||||
}
|
||||
@ -1057,7 +1022,7 @@ int main(int argc, const char **argv)
|
||||
|
||||
if (action == "import") {
|
||||
if (access(dir_name.c_str(), R_OK)) {
|
||||
cerr << argv[0] << ": source directory '" << dst
|
||||
cerr << "rados" << ": source directory '" << dst
|
||||
<< "' appears to be inaccessible." << std::endl;
|
||||
exit(ENOENT);
|
||||
}
|
||||
@ -1072,13 +1037,13 @@ int main(int argc, const char **argv)
|
||||
ret = mkdir(dst.c_str(), 0700);
|
||||
if (ret < 0) {
|
||||
ret = errno;
|
||||
cerr << argv[0] << ": mkdir(" << dst << ") failed with error " << ret
|
||||
cerr << "rados" << ": mkdir(" << dst << ") failed with error " << ret
|
||||
<< std::endl;
|
||||
exit(ret);
|
||||
}
|
||||
}
|
||||
else {
|
||||
cerr << argv[0] << ": directory '" << dst << "' is not accessible. Use "
|
||||
cerr << "rados" << ": directory '" << dst << "' is not accessible. Use "
|
||||
<< "--create to try to create it.\n";
|
||||
exit(ENOENT);
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@ die() {
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
test_rados_sync.sh: tests rados_sync
|
||||
test_rados_tool.sh: tests rados_tool
|
||||
-c: RADOS configuration file to use [optional]
|
||||
-k: keep temp files
|
||||
-h: this help message
|
||||
@ -34,11 +34,10 @@ run() {
|
||||
|
||||
DNAME="`dirname $0`"
|
||||
DNAME="`readlink -f $DNAME`"
|
||||
RADOS_SYNC="`readlink -f \"$DNAME/../rados_sync\"`"
|
||||
RADOS_TOOL="`readlink -f \"$DNAME/../rados\"`"
|
||||
KEEP_TEMP_FILES=0
|
||||
POOL=trs_pool
|
||||
[ -x "$RADOS_SYNC" ] || die "couldn't find $RADOS_SYNC binary to test"
|
||||
[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary to test"
|
||||
[ -x "$RADOS_TOOL" ] || die "couldn't find $RADOS_TOOL binary"
|
||||
which attr &>/dev/null
|
||||
[ $? -eq 0 ] || die "you must install the 'attr' tool to manipulate \
|
||||
@ -46,7 +45,7 @@ extended attributes."
|
||||
|
||||
while getopts "c:hkp:" flag; do
|
||||
case $flag in
|
||||
c) RADOS_SYNC="$RADOS_SYNC -c $OPTARG";;
|
||||
c) RADOS_TOOL="$RADOS_TOOL -c $OPTARG";;
|
||||
k) KEEP_TEMP_FILES=1;;
|
||||
h) usage; exit 0;;
|
||||
p) POOL=$OPTARG;;
|
||||
@ -54,7 +53,7 @@ while getopts "c:hkp:" flag; do
|
||||
esac
|
||||
done
|
||||
|
||||
TDIR=`mktemp -d -t test_rados_sync.XXXXXXXXXX` || die "mktemp failed"
|
||||
TDIR=`mktemp -d -t test_rados_tool.XXXXXXXXXX` || die "mktemp failed"
|
||||
[ $KEEP_TEMP_FILES -eq 0 ] && trap "rm -rf ${TDIR}; exit" INT TERM EXIT
|
||||
|
||||
mkdir "$TDIR/dira"
|
||||
@ -77,36 +76,36 @@ attr -q -s rados_full_name -V "foo2" "$TDIR/dirc/00003036_foo2"
|
||||
|
||||
# make sure that --create works
|
||||
run "$RADOS_TOOL" rmpool "$POOL"
|
||||
run_expect_succ "$RADOS_SYNC" --create import "$TDIR/dira" "$POOL"
|
||||
run_expect_succ "$RADOS_TOOL" --create import "$TDIR/dira" "$POOL"
|
||||
|
||||
# make sure that lack of --create fails
|
||||
run_expect_succ "$RADOS_TOOL" rmpool "$POOL"
|
||||
run_expect_fail "$RADOS_SYNC" import "$TDIR/dira" "$POOL"
|
||||
run_expect_fail "$RADOS_TOOL" import "$TDIR/dira" "$POOL"
|
||||
|
||||
run_expect_succ "$RADOS_SYNC" --create import "$TDIR/dira" "$POOL"
|
||||
run_expect_succ "$RADOS_TOOL" --create import "$TDIR/dira" "$POOL"
|
||||
|
||||
# inaccessible import src should fail
|
||||
run_expect_fail "$RADOS_SYNC" import "$TDIR/dir_nonexistent" "$POOL"
|
||||
run_expect_fail "$RADOS_TOOL" import "$TDIR/dir_nonexistent" "$POOL"
|
||||
|
||||
# export some stuff
|
||||
run_expect_succ "$RADOS_SYNC" --create export "$POOL" "$TDIR/dirb"
|
||||
run_expect_succ "$RADOS_TOOL" --create export "$POOL" "$TDIR/dirb"
|
||||
diff -q -r "$TDIR/dira" "$TDIR/dirb" \
|
||||
|| die "failed to export the same stuff we imported!"
|
||||
|
||||
# import some stuff with extended attributes on it
|
||||
run_expect_succ "$RADOS_SYNC" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out"
|
||||
run_expect_succ "$RADOS_TOOL" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out"
|
||||
run_expect_succ grep -q '\[xattr\]' $TDIR/out
|
||||
|
||||
# the second time, the xattrs should match, so there should be nothing to do.
|
||||
run_expect_succ "$RADOS_SYNC" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out"
|
||||
run_expect_succ "$RADOS_TOOL" import "$TDIR/dirc" "$POOL" | tee "$TDIR/out"
|
||||
run_expect_fail grep -q '\[xattr\]' "$TDIR/out"
|
||||
|
||||
# now force it to copy everything
|
||||
run_expect_succ "$RADOS_SYNC" --force import "$TDIR/dirc" "$POOL" | tee "$TDIR/out2"
|
||||
run_expect_succ "$RADOS_TOOL" --force import "$TDIR/dirc" "$POOL" | tee "$TDIR/out2"
|
||||
run_expect_succ grep '\[force\]' "$TDIR/out2"
|
||||
|
||||
# export some stuff with extended attributes on it
|
||||
run_expect_succ "$RADOS_SYNC" -C export "$POOL" "$TDIR/dirc_copy"
|
||||
run_expect_succ "$RADOS_TOOL" -C export "$POOL" "$TDIR/dirc_copy"
|
||||
|
||||
# check to make sure extended attributes were preserved
|
||||
PRE_EXPORT=`attr -qg rados.toothbrush "$TDIR/dirc/000029c4_foo"`
|
||||
@ -117,13 +116,13 @@ if [ "$PRE_EXPORT" != "$POST_EXPORT" ]; then
|
||||
fi
|
||||
|
||||
# trigger a rados delete using --delete-after
|
||||
run_expect_succ "$RADOS_SYNC" --create export "$POOL" "$TDIR/dird"
|
||||
run_expect_succ "$RADOS_TOOL" --create export "$POOL" "$TDIR/dird"
|
||||
rm -f "$TDIR/dird/000029c4_foo"
|
||||
run_expect_succ "$RADOS_SYNC" --delete-after import "$TDIR/dird" "$POOL" | tee "$TDIR/out3"
|
||||
run_expect_succ "$RADOS_TOOL" --delete-after import "$TDIR/dird" "$POOL" | tee "$TDIR/out3"
|
||||
run_expect_succ grep '\[deleted\]' "$TDIR/out3"
|
||||
|
||||
# trigger a local delete using --delete-after
|
||||
run_expect_succ "$RADOS_SYNC" --delete-after export "$POOL" "$TDIR/dirc" | tee "$TDIR/out4"
|
||||
run_expect_succ "$RADOS_TOOL" --delete-after export "$POOL" "$TDIR/dirc" | tee "$TDIR/out4"
|
||||
run_expect_succ grep '\[deleted\]' "$TDIR/out4"
|
||||
[ -e "$TDIR/dird/000029c4_foo" ] && die "--delete-after failed to delete a file!"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user