From da69fa09c8274585225471c68c1acc788a3881f4 Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Sun, 13 Oct 2013 13:40:57 +0100 Subject: [PATCH 1/3] tools: move 'test_store_tool' to 'tools/ceph-kvstore-tool' ceph-kvstore-tool allows for lower-level access to leveldb stores. Signed-off-by: Joao Eduardo Luis --- src/test/Makefile.am | 5 ----- src/tools/Makefile.am | 6 ++++++ .../test_store_tool.cc => tools/ceph-kvstore-tool.cc} | 0 3 files changed, 6 insertions(+), 5 deletions(-) rename src/{test/ObjectMap/test_store_tool/test_store_tool.cc => tools/ceph-kvstore-tool.cc} (100%) diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 59b4d89e930..84a228f1d4b 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -856,11 +856,6 @@ ceph_test_keyvaluedb_iterators_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) ceph_test_keyvaluedb_iterators_CXXFLAGS = $(UNITTEST_CXXFLAGS) bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_iterators -ceph_test_store_tool_SOURCES = test/ObjectMap/test_store_tool/test_store_tool.cc -ceph_test_store_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -ceph_test_store_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS) -bin_DEBUGPROGRAMS += ceph_test_store_tool - ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc bin_DEBUGPROGRAMS += ceph_test_cfuse_cache_invalidate diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am index 4b8da77951a..89417014dd4 100644 --- a/src/tools/Makefile.am +++ b/src/tools/Makefile.am @@ -6,6 +6,12 @@ ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options bin_DEBUGPROGRAMS += ceph-monstore-tool +ceph_kvstore_tool_SOURCES = tools/ceph-kvstore-tool.cc +ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) +ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS) +bin_DEBUGPROGRAMS += ceph-kvstore-tool + + ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options if LINUX diff --git a/src/test/ObjectMap/test_store_tool/test_store_tool.cc b/src/tools/ceph-kvstore-tool.cc similarity index 100% rename from src/test/ObjectMap/test_store_tool/test_store_tool.cc rename to src/tools/ceph-kvstore-tool.cc From 85914b27e67879e5d5b8f05c569919155b690d4f Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Sun, 13 Oct 2013 13:44:29 +0100 Subject: [PATCH 2/3] ceph-kvstore-tool: calc store crc Reuse 'list()' function to traverse the store and calc not only version's crcs, but also calc the store's crc. Signed-off-by: Joao Eduardo Luis --- src/tools/ceph-kvstore-tool.cc | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/tools/ceph-kvstore-tool.cc b/src/tools/ceph-kvstore-tool.cc index 8fcf3f30e82..ffd462bbabd 100644 --- a/src/tools/ceph-kvstore-tool.cc +++ b/src/tools/ceph-kvstore-tool.cc @@ -39,7 +39,9 @@ class StoreTool db.reset(db_ptr); } - void list(const string &prefix, const bool do_crc) { + uint32_t traverse(const string &prefix, + const bool do_crc, + ostream *out) { KeyValueDB::WholeSpaceIterator iter = db->get_iterator(); if (prefix.empty()) @@ -47,18 +49,36 @@ class StoreTool else iter->seek_to_first(prefix); + uint32_t crc = -1; + while (iter->valid()) { pair rk = iter->raw_key(); if (!prefix.empty() && (rk.first != prefix)) - break; + break; - std::cout << rk.first << ":" << rk.second; + if (out) + *out << rk.first << ":" << rk.second; if (do_crc) { - std::cout << " (" << iter->value().crc32c(0) << ")"; + bufferlist bl; + bl.append(rk.first); + bl.append(rk.second); + bl.append(iter->value()); + + crc = bl.crc32c(crc); + if (out) { + *out << " (" << bl.crc32c(0) << ")"; + } } - std::cout << std::endl; + if (out) + *out << std::endl; iter->next(); } + + return crc; + } + + void list(const string &prefix, const bool do_crc) { + traverse(prefix, do_crc, &std::cout); } bool exists(const string &prefix) { @@ -132,6 +152,7 @@ void usage(const char *pname) << " crc \n" << " get-size\n" << " set [ver |in ]\n" + << " store-crc \n" << std::endl; } @@ -260,6 +281,9 @@ int main(int argc, const char *argv[]) << prefix << "," << key << ")" << std::endl; return 1; } + } else if (cmd == "store-crc") { + uint32_t crc = st.traverse(string(), true, NULL); + std::cout << "store at '" << path << "' crc " << crc << std::endl; } else { std::cerr << "Unrecognized command: " << cmd << std::endl; From fd6e2b86188845e6c115d65a5cfaeb60f57c6a3a Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Sun, 13 Oct 2013 13:45:43 +0100 Subject: [PATCH 3/3] ceph-kvstore-tool: copy one leveldb store to some other place Iterates over the provided source store's keys and copies them to the provided destination store. Signed-off-by: Joao Eduardo Luis --- src/tools/ceph-kvstore-tool.cc | 94 ++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/src/tools/ceph-kvstore-tool.cc b/src/tools/ceph-kvstore-tool.cc index ffd462bbabd..e07391d5c51 100644 --- a/src/tools/ceph-kvstore-tool.cc +++ b/src/tools/ceph-kvstore-tool.cc @@ -25,16 +25,18 @@ #include "common/safe_io.h" #include "common/config.h" #include "common/strtol.h" +#include "include/stringify.h" using namespace std; class StoreTool { boost::scoped_ptr db; + string store_path; public: - StoreTool(const string &path) { - LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, path); + StoreTool(const string &path) : store_path(path) { + LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path); assert(!db_ptr->open(std::cerr)); db.reset(db_ptr); } @@ -138,6 +140,70 @@ class StoreTool return (ret == 0); } + + int copy_store_to(const string &other_path, const int num_keys_per_tx) { + + if (num_keys_per_tx <= 0) { + std::cerr << "must specify a number of keys/tx > 0" << std::endl; + return -EINVAL; + } + + // open or create a leveldb store at @p other_path + LevelDBStore other(g_ceph_context, other_path); + int err = other.create_and_open(std::cerr); + if (err < 0) + return err; + + KeyValueDB::WholeSpaceIterator it = db->get_iterator(); + it->seek_to_first(); + uint64_t total_keys = 0; + uint64_t total_size = 0; + uint64_t total_txs = 0; + + utime_t started_at = ceph_clock_now(g_ceph_context); + + do { + int num_keys = 0; + + KeyValueDB::Transaction tx = other.get_transaction(); + + + while (it->valid() && num_keys < num_keys_per_tx) { + pair k = it->raw_key(); + bufferlist v = it->value(); + tx->set(k.first, k.second, v); + + num_keys ++; + total_size += v.length(); + + it->next(); + } + + total_txs ++; + total_keys += num_keys; + + if (num_keys > 0) + other.submit_transaction_sync(tx); + + utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at; + std::cout << "ts = " << cur_duration << "s, copied " << total_keys + << " keys so far (" << stringify(si_t(total_size)) << ")" + << std::endl; + + } while (it->valid()); + + utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at; + + std::cout << "summary:" << std::endl; + std::cout << " copied " << total_keys << " keys" << std::endl; + std::cout << " used " << total_txs << " transactions" << std::endl; + std::cout << " total size " << stringify(si_t(total_size)) << std::endl; + std::cout << " from '" << store_path << "' to '" << other_path << "'" + << std::endl; + std::cout << " duration " << time_taken << " seconds" << std::endl; + + return 0; + } }; void usage(const char *pname) @@ -152,6 +218,7 @@ void usage(const char *pname) << " crc \n" << " get-size\n" << " set [ver |in ]\n" + << " store-copy [num-keys-per-tx]\n" << " store-crc \n" << std::endl; } @@ -176,8 +243,6 @@ int main(int argc, const char *argv[]) string path(args[0]); string cmd(args[1]); - std::cout << "path: " << path << " cmd " << cmd << std::endl; - StoreTool st(path); if (cmd == "list" || cmd == "list-crc") { @@ -281,6 +346,27 @@ int main(int argc, const char *argv[]) << prefix << "," << key << ")" << std::endl; return 1; } + } else if (cmd == "store-copy") { + int num_keys_per_tx = 128; // magic number that just feels right. + if (argc < 4) { + usage(argv[0]); + return 1; + } else if (argc > 4) { + string err; + num_keys_per_tx = strict_strtol(argv[4], 10, &err); + if (!err.empty()) { + std::cerr << "invalid num_keys_per_tx: " << err << std::endl; + return 1; + } + } + + int ret = st.copy_store_to(argv[3], num_keys_per_tx); + if (ret < 0) { + std::cerr << "error copying store to path '" << argv[3] + << "': " << cpp_strerror(ret) << std::endl; + return 1; + } + } else if (cmd == "store-crc") { uint32_t crc = st.traverse(string(), true, NULL); std::cout << "store at '" << path << "' crc " << crc << std::endl;