Merge pull request #732 from ceph/wip-kvstore-tool

tools: ceph-kvstore-tool: Access & manipulate leveldb store using the KeyValueDB interface

Reviewed-by: Sage Weil <sage@inktank.com>

src/test/Makefile.am

@@ -856,11 +856,6 @@ ceph_test_keyvaluedb_iterators_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL)
 ceph_test_keyvaluedb_iterators_CXXFLAGS = $(UNITTEST_CXXFLAGS)
 bin_DEBUGPROGRAMS += ceph_test_keyvaluedb_iterators
 
-ceph_test_store_tool_SOURCES = test/ObjectMap/test_store_tool/test_store_tool.cc
-ceph_test_store_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL)
-ceph_test_store_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
-bin_DEBUGPROGRAMS += ceph_test_store_tool
-
 ceph_test_cfuse_cache_invalidate_SOURCES = test/test_cfuse_cache_invalidate.cc
 bin_DEBUGPROGRAMS += ceph_test_cfuse_cache_invalidate
 

src/tools/Makefile.am

@@ -6,6 +6,12 @@ ceph_monstore_tool_SOURCES = tools/ceph-monstore-tool.cc
 ceph_monstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
 bin_DEBUGPROGRAMS += ceph-monstore-tool
 
+ceph_kvstore_tool_SOURCES = tools/ceph-kvstore-tool.cc
+ceph_kvstore_tool_LDADD = $(LIBOS) $(CEPH_GLOBAL)
+ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
+bin_DEBUGPROGRAMS += ceph-kvstore-tool
+
+
 ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
 ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
 if LINUX

src/tools/ceph-kvstore-tool.cc

@@ -25,21 +25,25 @@
 #include "common/safe_io.h"
 #include "common/config.h"
 #include "common/strtol.h"
+#include "include/stringify.h"
 
 using namespace std;
 
 class StoreTool
 {
   boost::scoped_ptr<KeyValueDB> db;
+  string store_path;
 
   public:
-  StoreTool(const string &path) {
-    LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, path);
+  StoreTool(const string &path) : store_path(path) {
+    LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, store_path);
     assert(!db_ptr->open(std::cerr));
     db.reset(db_ptr);
   }
 
-  void list(const string &prefix, const bool do_crc) {
+  uint32_t traverse(const string &prefix,
+                    const bool do_crc,
+                    ostream *out) {
     KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
 
     if (prefix.empty())
@@ -47,18 +51,36 @@ class StoreTool
     else
       iter->seek_to_first(prefix);
 
+    uint32_t crc = -1;
+
     while (iter->valid()) {
       pair<string,string> rk = iter->raw_key();
       if (!prefix.empty() && (rk.first != prefix))
-	break;
+        break;
 
-      std::cout << rk.first << ":" << rk.second;
+      if (out)
+        *out << rk.first << ":" << rk.second;
       if (do_crc) {
-        std::cout << " (" << iter->value().crc32c(0) << ")";
+        bufferlist bl;
+        bl.append(rk.first);
+        bl.append(rk.second);
+        bl.append(iter->value());
+
+        crc = bl.crc32c(crc);
+        if (out) {
+          *out << " (" << bl.crc32c(0) << ")";
+        }
       }
-      std::cout << std::endl;
+      if (out)
+        *out << std::endl;
       iter->next();
     }
+
+    return crc;
+  }
+
+  void list(const string &prefix, const bool do_crc) {
+    traverse(prefix, do_crc, &std::cout);
   }
 
   bool exists(const string &prefix) {
@@ -118,6 +140,70 @@ class StoreTool
 
     return (ret == 0);
   }
+
+  int copy_store_to(const string &other_path, const int num_keys_per_tx) {
+
+    if (num_keys_per_tx <= 0) {
+      std::cerr << "must specify a number of keys/tx > 0" << std::endl;
+      return -EINVAL;
+    }
+
+    // open or create a leveldb store at @p other_path
+    LevelDBStore other(g_ceph_context, other_path);
+    int err = other.create_and_open(std::cerr);
+    if (err < 0)
+      return err;
+
+    KeyValueDB::WholeSpaceIterator it = db->get_iterator();
+    it->seek_to_first();
+    uint64_t total_keys = 0;
+    uint64_t total_size = 0;
+    uint64_t total_txs = 0;
+
+    utime_t started_at = ceph_clock_now(g_ceph_context);
+
+    do {
+      int num_keys = 0;
+
+      KeyValueDB::Transaction tx = other.get_transaction();
+
+
+      while (it->valid() && num_keys < num_keys_per_tx) {
+        pair<string,string> k = it->raw_key();
+        bufferlist v = it->value();
+        tx->set(k.first, k.second, v);
+
+        num_keys ++;
+        total_size += v.length();
+
+        it->next();
+      }
+
+      total_txs ++;
+      total_keys += num_keys;
+
+      if (num_keys > 0)
+        other.submit_transaction_sync(tx);
+
+      utime_t cur_duration = ceph_clock_now(g_ceph_context) - started_at;
+      std::cout << "ts = " << cur_duration << "s, copied " << total_keys
+                << " keys so far (" << stringify(si_t(total_size)) << ")"
+                << std::endl;
+
+    } while (it->valid());
+
+    utime_t time_taken = ceph_clock_now(g_ceph_context) - started_at;
+
+    std::cout << "summary:" << std::endl;
+    std::cout << "  copied " << total_keys << " keys" << std::endl;
+    std::cout << "  used " << total_txs << " transactions" << std::endl;
+    std::cout << "  total size " << stringify(si_t(total_size)) << std::endl;
+    std::cout << "  from '" << store_path << "' to '" << other_path << "'"
+              << std::endl;
+    std::cout << "  duration " << time_taken << " seconds" << std::endl;
+
+    return 0;
+  }
 };
 
 void usage(const char *pname)
@@ -132,6 +218,8 @@ void usage(const char *pname)
     << "  crc <prefix> <key>\n"
     << "  get-size\n"
     << "  set <prefix> <key> [ver <N>|in <file>]\n"
+    << "  store-copy <path> [num-keys-per-tx]\n"
+    << "  store-crc <path>\n"
     << std::endl;
 }
 
@@ -155,8 +243,6 @@ int main(int argc, const char *argv[])
   string path(args[0]);
   string cmd(args[1]);
 
-  std::cout << "path: " << path << " cmd " << cmd << std::endl;
-
   StoreTool st(path);
 
   if (cmd == "list" || cmd == "list-crc") {
@@ -260,6 +346,30 @@ int main(int argc, const char *argv[])
                 << prefix << "," << key << ")" << std::endl;
       return 1;
     }
+  } else if (cmd == "store-copy") {
+    int num_keys_per_tx = 128; // magic number that just feels right.
+    if (argc < 4) {
+      usage(argv[0]);
+      return 1;
+    } else if (argc > 4) {
+      string err;
+      num_keys_per_tx = strict_strtol(argv[4], 10, &err);
+      if (!err.empty()) {
+        std::cerr << "invalid num_keys_per_tx: " << err << std::endl;
+        return 1;
+      }
+    }
+
+    int ret = st.copy_store_to(argv[3], num_keys_per_tx);
+    if (ret < 0) {
+      std::cerr << "error copying store to path '" << argv[3]
+                << "': " << cpp_strerror(ret) << std::endl;
+      return 1;
+    }
+
+  } else if (cmd == "store-crc") {
+    uint32_t crc = st.traverse(string(), true, NULL);
+    std::cout << "store at '" << path << "' crc " << crc << std::endl;
 
   } else {
     std::cerr << "Unrecognized command: " << cmd << std::endl;