Merge remote-tracking branch 'upstream/next'

ceph.spec.in

@@ -625,6 +625,7 @@ fi
 %{_bindir}/ceph_smalliobenchfs
 %{_bindir}/ceph_smalliobenchrbd
 %{_bindir}/ceph_filestore_dump
+%{_bindir}/ceph_filestore_tool
 %{_bindir}/ceph_streamtest
 %{_bindir}/ceph_test_cfuse_cache_invalidate
 %{_bindir}/ceph_test_cls_hello

debian/ceph-test.install

@@ -2,6 +2,7 @@ usr/bin/ceph-coverage
 usr/bin/ceph_bench_log
 usr/bin/ceph_dupstore
 usr/bin/ceph_filestore_dump
+usr/bin/ceph_filestore_tool
 usr/bin/ceph_kvstorebench
 usr/bin/ceph_multi_stress_watch
 usr/bin/ceph_omapbench

src/.gitignore

@@ -24,6 +24,7 @@ Makefile
 /ceph_bench_log
 /ceph_dupstore
 /ceph_filestore_dump
+/ceph_filestore_tool
 /ceph_mon_store_converter
 /ceph_multi_stress_watch
 /ceph_psim

src/common/ceph_context.cc

@@ -292,12 +292,12 @@ CephContext::CephContext(uint32_t module_type_)
 
 CephContext::~CephContext()
 {
+  join_service_thread();
+
   if (_conf->lockdep) {
     lockdep_unregister_ceph_context(this);
   }
 
-  join_service_thread();
-
   _admin_socket->unregister_command("perfcounters_dump");
   _admin_socket->unregister_command("perf dump");
   _admin_socket->unregister_command("1");

src/common/hobject.h

@@ -98,6 +98,11 @@ public:
     return ret;
   }
 
+  /// @return true if object is head
+  bool is_head() const {
+    return snap == CEPH_NOSNAP;
+  }
+
   /// @return true if object is neither head nor snapdir
   bool is_snap() const {
     return (snap != CEPH_NOSNAP) && (snap != CEPH_SNAPDIR);

src/os/FileJournal.cc

@@ -469,7 +469,6 @@ int FileJournal::open(uint64_t fs_op_seq)
 {
   dout(2) << "open " << fn << " fsid " << fsid << " fs_op_seq " << fs_op_seq << dendl;
 
-  last_committed_seq = fs_op_seq;
   uint64_t next_seq = fs_op_seq + 1;
 
   int err = _open(false);
@@ -528,6 +527,11 @@ int FileJournal::open(uint64_t fs_op_seq)
   // find next entry
   read_pos = header.start;
   uint64_t seq = header.start_seq;
+
+  // last_committed_seq is 1 before the start of the journal or
+  // 0 if the start is 0
+  last_committed_seq = seq > 0 ? seq - 1 : seq;
+
   while (1) {
     bufferlist bl;
     off64_t old_pos = read_pos;

src/os/JournalingObjectStore.cc

@@ -100,6 +100,8 @@ int JournalingObjectStore::journal_replay(uint64_t fs_op_seq)
   // done reading, make writeable.
   journal->make_writeable();
 
+  journal->committed_thru(fs_op_seq);
+
   return count;
 }
 

src/osd/ReplicatedPG.cc

@@ -1046,9 +1046,12 @@ void ReplicatedPG::do_op(OpRequestRef op)
       if (!src_obc.count(src_oid)) {
 	ObjectContextRef sobc;
 	snapid_t ssnapid;
+	int r;
 
-	int r = find_object_context(src_oid, &sobc, false, &ssnapid);
-	if (r == -EAGAIN) {
+	if (src_oid.is_head() && is_missing_object(src_oid)) {
+	  wait_for_missing_object(src_oid, op);
+	} else if ((r = find_object_context(
+		      src_oid, &sobc, false, &ssnapid)) == -EAGAIN) {
 	  // missing the specific snap we need; requeue and wait.
 	  hobject_t wait_oid(osd_op.soid.oid, src_oloc.key, ssnapid, m->get_pg().ps(),
 			     info.pgid.pool(), m->get_object_locator().nspace);

src/osd/osd_types.cc

@@ -2913,7 +2913,7 @@ void object_info_t::encode(bufferlist& bl) const
        ++i) {
     old_watchers.insert(make_pair(i->first.second, i->second));
   }
-  ENCODE_START(12, 8, bl);
+  ENCODE_START(13, 8, bl);
   ::encode(soid, bl);
   ::encode(myoloc, bl);	//Retained for compatibility
   ::encode(category, bl);
@@ -2928,23 +2928,23 @@ void object_info_t::encode(bufferlist& bl) const
     ::encode(snaps, bl);
   ::encode(truncate_seq, bl);
   ::encode(truncate_size, bl);
-  __u8 flags_lo = flags & 0xff;
-  __u8 flags_hi = (flags & 0xff00) >> 8;
-  ::encode(flags_lo, bl);
+  ::encode(is_lost(), bl);
   ::encode(old_watchers, bl);
   /* shenanigans to avoid breaking backwards compatibility in the disk format.
    * When we can, switch this out for simply putting the version_t on disk. */
   eversion_t user_eversion(0, user_version);
   ::encode(user_eversion, bl);
-  ::encode(flags_hi, bl);
+  ::encode(test_flag(FLAG_USES_TMAP), bl);
   ::encode(watchers, bl);
+  __u32 _flags = flags;
+  ::encode(_flags, bl);
   ENCODE_FINISH(bl);
 }
 
 void object_info_t::decode(bufferlist::iterator& bl)
 {
   object_locator_t myoloc;
-  DECODE_START_LEGACY_COMPAT_LEN(12, 8, 8, bl);
+  DECODE_START_LEGACY_COMPAT_LEN(13, 8, 8, bl);
   map<entity_name_t, watch_info_t> old_watchers;
   if (struct_v >= 2 && struct_v <= 5) {
     sobject_t obj;
@@ -2975,6 +2975,9 @@ void object_info_t::decode(bufferlist::iterator& bl)
   ::decode(truncate_seq, bl);
   ::decode(truncate_size, bl);
   if (struct_v >= 3) {
+    // if this is struct_v >= 13, we will overwrite this
+    // below since this field is just here for backwards
+    // compatibility
     __u8 lo;
     ::decode(lo, bl);
     flags = (flag_t)lo;
@@ -2988,9 +2991,10 @@ void object_info_t::decode(bufferlist::iterator& bl)
     user_version = user_eversion.version;
   }
   if (struct_v >= 9) {
-    __u8 hi;
-    ::decode(hi, bl);
-    flags = (flag_t)(flags | ((unsigned)hi << 8));
+    bool uses_tmap = false;
+    ::decode(uses_tmap, bl);
+    if (uses_tmap)
+      set_flag(FLAG_USES_TMAP);
   } else {
     set_flag(FLAG_USES_TMAP);
   }
@@ -3007,6 +3011,11 @@ void object_info_t::decode(bufferlist::iterator& bl)
 	  make_pair(i->second.cookie, i->first), i->second));
     }
   }
+  if (struct_v >= 13) {
+    __u32 _flags;
+    ::decode(_flags, bl);
+    flags = (flag_t)_flags;
+  }
   DECODE_FINISH(bl);
 }
 

src/tools/Makefile.am

@@ -12,6 +12,13 @@ ceph_kvstore_tool_CXXFLAGS = $(UNITTEST_CXXFLAGS)
 bin_DEBUGPROGRAMS += ceph-kvstore-tool
 
 
+ceph_filestore_tool_SOURCES = tools/ceph-filestore-tool.cc
+ceph_filestore_tool_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
+if LINUX
+ceph_filestore_tool_LDADD += -ldl
+endif # LINUX
+bin_PROGRAMS += ceph_filestore_tool
+
 ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
 ceph_filestore_dump_LDADD = $(LIBOSD) $(LIBOS) $(CEPH_GLOBAL) -lboost_program_options
 if LINUX

src/tools/ceph-filestore-tool.cc

@@ -0,0 +1,260 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Inktank
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/program_options/option.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/program_options/cmdline.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <iostream>
+#include <set>
+#include <sstream>
+#include <stdlib.h>
+#include <fstream>
+
+#include "common/Formatter.h"
+
+#include "global/global_init.h"
+#include "os/ObjectStore.h"
+#include "os/FileStore.h"
+#include "common/perf_counters.h"
+#include "common/errno.h"
+#include "osd/PGLog.h"
+#include "osd/osd_types.h"
+#include "osd/OSD.h"
+
+namespace po = boost::program_options;
+using namespace std;
+
+static void invalid_path(string &path)
+{
+  cout << "Invalid path to osd store specified: " << path << "\n";
+  exit(1);
+}
+
+int main(int argc, char **argv)
+{
+  string fspath, jpath, pgidstr;
+  bool list_lost_objects = false;
+  bool fix_lost_objects = false;
+  unsigned LIST_AT_A_TIME = 100;
+  unsigned scanned = 0;
+  
+  po::options_description desc("Allowed options");
+  desc.add_options()
+    ("help", "produce help message")
+    ("filestore-path", po::value<string>(&fspath),
+     "path to filestore directory, mandatory")
+    ("journal-path", po::value<string>(&jpath),
+     "path to journal, mandatory")
+    ("pgid", po::value<string>(&pgidstr),
+     "PG id")
+    ("list-lost-objects", po::value<bool>(
+      &list_lost_objects)->default_value(false),
+     "list lost objects")
+    ("fix-lost-objects", po::value<bool>(
+      &fix_lost_objects)->default_value(false),
+     "fix lost objects")
+    ;
+
+  po::variables_map vm;
+  po::parsed_options parsed =
+   po::command_line_parser(argc, argv).options(desc).
+    allow_unregistered().run();
+  po::store( parsed, vm);
+  try {
+    po::notify(vm);
+  }
+  catch(...) {
+    cout << desc << std::endl;
+    exit(1);
+  }
+     
+  if (vm.count("help")) {
+    cout << desc << std::endl;
+    return 1;
+  }
+
+  if (!vm.count("filestore-path")) {
+    cerr << "Must provide filestore-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+  if (!vm.count("journal-path")) {
+    cerr << "Must provide journal-path" << std::endl
+	 << desc << std::endl;
+    return 1;
+  } 
+
+  if ((fspath.length() == 0 || jpath.length() == 0)) {
+    cerr << "Invalid params" << desc << std::endl;
+    exit(1);
+  }
+
+  vector<const char *> ceph_options, def_args;
+  vector<string> ceph_option_strings = po::collect_unrecognized(
+    parsed.options, po::include_positional);
+  ceph_options.reserve(ceph_option_strings.size());
+  for (vector<string>::iterator i = ceph_option_strings.begin();
+       i != ceph_option_strings.end();
+       ++i) {
+    ceph_options.push_back(i->c_str());
+  }
+
+  global_init(
+    &def_args, ceph_options, CEPH_ENTITY_TYPE_OSD,
+    CODE_ENVIRONMENT_UTILITY, 0);
+    //CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
+  common_init_finish(g_ceph_context);
+  g_ceph_context->_conf->apply_changes(NULL);
+  g_conf = g_ceph_context->_conf;
+
+  //Verify that fspath really is an osd store
+  struct stat st;
+  if (::stat(fspath.c_str(), &st) == -1) {
+     perror("fspath");
+     invalid_path(fspath);
+  }
+  if (!S_ISDIR(st.st_mode)) {
+    invalid_path(fspath);
+  }
+  string check = fspath + "/whoami";
+  if (::stat(check.c_str(), &st) == -1) {
+     perror("whoami");
+     invalid_path(fspath);
+  }
+  if (!S_ISREG(st.st_mode)) {
+    invalid_path(fspath);
+  }
+  check = fspath + "/current";
+  if (::stat(check.c_str(), &st) == -1) {
+     perror("current");
+     invalid_path(fspath);
+  }
+  if (!S_ISDIR(st.st_mode)) {
+    invalid_path(fspath);
+  }
+
+  ObjectStore *fs = new FileStore(fspath, jpath);
+
+  int r = fs->mount();
+  if (r < 0) {
+    if (r == -EBUSY) {
+      cout << "OSD has the store locked" << std::endl;
+    } else {
+      cout << "Mount failed with '" << cpp_strerror(-r) << "'" << std::endl;
+    }
+    return 1;
+  }
+
+  vector<coll_t> colls_to_check;
+  if (pgidstr.length()) {
+    pg_t pgid;
+    if (!pgid.parse(pgidstr.c_str())) {
+      cout << "Invalid pgid '" << pgidstr << "' specified" << std::endl;
+      exit(1);
+    }
+    colls_to_check.push_back(coll_t(pgid));
+  } else {
+    vector<coll_t> candidates;
+    r = fs->list_collections(candidates);
+    if (r < 0) {
+      cerr << "Error listing collections: " << cpp_strerror(r) << std::endl;
+      goto UMOUNT;
+    }
+    for (vector<coll_t>::iterator i = candidates.begin();
+	 i != candidates.end();
+	 ++i) {
+      pg_t pgid;
+      snapid_t snap;
+      if (i->is_pg(pgid, snap)) {
+	colls_to_check.push_back(*i);
+      }
+    }
+  }
+
+  cerr << colls_to_check.size() << " pgs to scan" << std::endl;
+  for (vector<coll_t>::iterator i = colls_to_check.begin();
+       i != colls_to_check.end();
+       ++i, ++scanned) {
+    cerr << "Scanning " << *i << ", " << scanned << "/"
+	 << colls_to_check.size() << " completed" << std::endl;
+    ghobject_t next;
+    while (!next.is_max()) {
+      vector<ghobject_t> list;
+      r = fs->collection_list_partial(
+	*i,
+	next,
+	LIST_AT_A_TIME,
+	LIST_AT_A_TIME,
+	CEPH_NOSNAP,
+	&list,
+	&next);
+      if (r < 0) {
+	cerr << "Error listing collection: " << *i << ", "
+	     << cpp_strerror(r) << std::endl;
+	goto UMOUNT;
+      }
+      for (vector<ghobject_t>::iterator obj = list.begin();
+	   obj != list.end();
+	   ++obj) {
+	bufferlist attr;
+	r = fs->getattr(*i, *obj, OI_ATTR, attr);
+	if (r < 0) {
+	  cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+	       << cpp_strerror(r) << std::endl;
+	  goto UMOUNT;
+	}
+	object_info_t oi;
+	bufferlist::iterator bp = attr.begin();
+	try {
+	  ::decode(oi, bp);
+	} catch (...) {
+	  r = -EINVAL;
+	  cerr << "Error getting attr on : " << make_pair(*i, *obj) << ", "
+	       << cpp_strerror(r) << std::endl;
+	  goto UMOUNT;
+	}
+	if (oi.is_lost()) {
+	  if (list_lost_objects) {
+	    cout << *i << "/" << *obj << " is lost" << std::endl;
+	  }
+	  if (fix_lost_objects) {
+	    cerr << *i << "/" << *obj << " is lost, fixing" << std::endl;
+	    oi.clear_flag(object_info_t::FLAG_LOST);
+	    bufferlist bl2;
+	    ::encode(oi, bl2);
+	    ObjectStore::Transaction t;
+	    t.setattr(*i, *obj, OI_ATTR, bl2);
+	    r = fs->apply_transaction(t);
+	    if (r < 0) {
+	      cerr << "Error getting fixing attr on : " << make_pair(*i, *obj)
+		   << ", "
+		   << cpp_strerror(r) << std::endl;
+	      goto UMOUNT;
+	    }
+	  }
+	}
+      }
+    }
+  }
+  cerr << "Completed" << std::endl;
+
+ UMOUNT:
+  fs->sync_and_flush();
+  fs->umount();
+  return r;
+}