Merge pull request #3704 from ceph/wip-10761

librbd: blacklist clients before breaking locks

Reviewed-by: Josh Durgin <jdurgin@redhat.com>

src/common/config_opts.h

@@ -864,6 +864,8 @@ OPTION(rbd_readahead_max_bytes, OPT_LONGLONG, 512 * 1024) // set to 0 to disable
 OPTION(rbd_readahead_disable_after_bytes, OPT_LONGLONG, 50 * 1024 * 1024) // how many bytes are read in total before readahead is disabled
 OPTION(rbd_clone_copy_on_read, OPT_BOOL, false)
 OPTION(rbd_object_map, OPT_BOOL, false) // whether to enable the RBD object map
+OPTION(rbd_blacklist_on_break_lock, OPT_BOOL, true) // whether to blacklist clients whose lock was broken
+OPTION(rbd_blacklist_expire_seconds, OPT_INT, 0) // number of seconds to blacklist - set to 0 for OSD default
 
 /*
  * The following options change the behavior for librbd's image creation methods that

src/include/rados/librados.h

@@ -2797,6 +2797,19 @@ CEPH_RADOS_API ssize_t rados_list_lockers(rados_ioctx_t io, const char *o,
 CEPH_RADOS_API int rados_break_lock(rados_ioctx_t io, const char *o,
                                     const char *name, const char *client,
                                     const char *cookie);
+
+/**
+ * Blacklists the specified client from the OSDs
+ *
+ * @param cluster cluster handle
+ * @param client_address client address
+ * @param expire_seconds number of seconds to blacklist (0 for default)
+ * @returns 0 on success, negative error code on failure
+ */
+CEPH_RADOS_API int rados_blacklist_add(rados_t cluster,
+				       char *client_address,
+				       uint32_t expire_seconds);
+
 /**
  * @defgroup librados_h_commands Mon/OSD/PG Commands
  *

src/include/rados/librados.hpp

@@ -1108,6 +1108,9 @@ namespace librados
     /// get/wait for the most recent osdmap
     int wait_for_latest_osdmap();
 
+    int blacklist_add(const std::string& client_address,
+                      uint32_t expire_seconds);
+
     /*
      * pool aio
      *

src/librados/RadosClient.cc

@@ -18,6 +18,7 @@
 
 #include <iostream>
 #include <string>
+#include <sstream>
 #include <pthread.h>
 #include <errno.h>
 
@@ -656,6 +657,32 @@ void librados::RadosClient::blacklist_self(bool set) {
   objecter->blacklist_self(set);
 }
 
+int librados::RadosClient::blacklist_add(const string& client_address,
+					 uint32_t expire_seconds)
+{
+  entity_addr_t addr;
+  if (!addr.parse(client_address.c_str(), 0)) {
+    lderr(cct) << "unable to parse address " << client_address << dendl;
+    return -EINVAL;
+  }
+
+  std::stringstream cmd;
+  cmd << "{"
+      << "\"prefix\": \"osd blacklist\", "
+      << "\"blacklistop\": \"add\", "
+      << "\"addr\": \"" << client_address << "\"";
+  if (expire_seconds != 0) {
+    cmd << ", \"expire\": " << expire_seconds << ".0";
+  }
+  cmd << "}";
+
+  std::vector<std::string> cmds;
+  cmds.push_back(cmd.str());
+  bufferlist inbl;
+  int r = mon_command(cmds, inbl, NULL, NULL);
+  return r;
+}
+
 int librados::RadosClient::mon_command(const vector<string>& cmd,
 				       const bufferlist &inbl,
 				       bufferlist *outbl, string *outs)

src/librados/RadosClient.h

@@ -109,6 +109,8 @@ public:
 
   int pool_delete_async(const char *name, PoolAsyncCompletionImpl *c);
 
+  int blacklist_add(const string& client_address, uint32_t expire_seconds);
+
   int mon_command(const vector<string>& cmd, const bufferlist &inbl,
 	          bufferlist *outbl, string *outs);
   int mon_command(int rank,

src/librados/librados.cc

@@ -2126,6 +2126,12 @@ int librados::Rados::wait_for_latest_osdmap()
   return client->wait_for_latest_osdmap();
 }
 
+int librados::Rados::blacklist_add(const std::string& client_address,
+				   uint32_t expire_seconds)
+{
+  return client->blacklist_add(client_address, expire_seconds);
+}
+
 librados::PoolAsyncCompletion *librados::Rados::pool_async_create_completion()
 {
   PoolAsyncCompletionImpl *c = new PoolAsyncCompletionImpl;
@@ -2471,6 +2477,13 @@ extern "C" int rados_wait_for_latest_osdmap(rados_t cluster)
   return retval;
 }
 
+extern "C" int rados_blacklist_add(rados_t cluster, char *client_address,
+				   uint32_t expire_seconds)
+{
+  librados::RadosClient *radosp = (librados::RadosClient *)cluster;
+  return radosp->blacklist_add(client_address, expire_seconds);
+}
+
 extern "C" int rados_pool_list(rados_t cluster, char *buf, size_t len)
 {
   tracepoint(librados, rados_pool_list_enter, cluster, len);

src/librbd/ImageWatcher.cc

@@ -184,6 +184,20 @@ int ImageWatcher::try_lock() {
       }
     }
 
+    md_config_t *conf = m_image_ctx.cct->_conf;
+    if (conf->rbd_blacklist_on_break_lock) {
+      ldout(m_image_ctx.cct, 1) << "blacklisting client: " << locker << "@"
+				<< locker_address << dendl;
+      librados::Rados rados(m_image_ctx.md_ctx);
+      r = rados.blacklist_add(locker_address,
+			      conf->rbd_blacklist_expire_seconds);
+      if (r < 0) {
+        lderr(m_image_ctx.cct) << "unable to blacklist client: "
+			       << cpp_strerror(r) << dendl;
+        return r;
+      }
+    }
+
     ldout(m_image_ctx.cct, 1) << "breaking exclusive lock: " << locker << dendl;
     r = rados::cls::lock::break_lock(&m_image_ctx.md_ctx,
                                      m_image_ctx.header_oid, RBD_LOCK_NAME,

src/librbd/internal.cc

@@ -2597,7 +2597,46 @@ reprotect_and_return_err:
 		       << "'" << dendl;
       return -EINVAL;
     }
-    RWLock::RLocker locker(ictx->md_lock);
+
+    md_config_t *conf = ictx->cct->_conf;
+    if (conf->rbd_blacklist_on_break_lock) {
+      typedef std::map<rados::cls::lock::locker_id_t,
+		       rados::cls::lock::locker_info_t> Lockers;
+      Lockers lockers;
+      ClsLockType lock_type;
+      std::string lock_tag;
+      r = rados::cls::lock::get_lock_info(&ictx->md_ctx, ictx->header_oid,
+                                          RBD_LOCK_NAME, &lockers, &lock_type,
+                                          &lock_tag);
+      if (r < 0) {
+        lderr(ictx->cct) << "unable to retrieve lock info: " << cpp_strerror(r)
+          	       << dendl;
+        return r;
+      }
+
+      std::string client_address;
+      for (Lockers::iterator it = lockers.begin();
+           it != lockers.end(); ++it) {
+        if (it->first.locker == lock_client) {
+          client_address = stringify(it->second.addr);
+          break;
+        }
+      }
+      if (client_address.empty()) {
+        return -ENOENT;
+      }
+      
+      RWLock::RLocker locker(ictx->md_lock);
+      librados::Rados rados(ictx->md_ctx);
+      r = rados.blacklist_add(client_address,
+			      conf->rbd_blacklist_expire_seconds);
+      if (r < 0) {
+        lderr(ictx->cct) << "unable to blacklist client: " << cpp_strerror(r)
+          	       << dendl;
+        return r;
+      }
+    }
+
     r = rados::cls::lock::break_lock(&ictx->md_ctx, ictx->header_oid,
 				     RBD_LOCK_NAME, cookie, lock_client);
     if (r < 0)

src/pybind/rados.py

@@ -5,7 +5,7 @@ Copyright 2011, Hannu Valtonen <hannu.valtonen@ormod.com>
 """
 from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_char, c_int, c_long, \
     c_ulong, create_string_buffer, byref, Structure, c_uint64, c_ubyte, \
-    pointer, CFUNCTYPE, c_int64, c_uint8
+    pointer, CFUNCTYPE, c_int64, c_uint32, c_uint8
 from ctypes.util import find_library
 import ctypes
 import errno
@@ -766,6 +766,24 @@ Rados object in state %s." % self.state)
         self.require_state("connected")
         return run_in_thread(self.librados.rados_wait_for_latest_osdmap, (self.cluster,))
 
+    def blacklist_add(self, client_address, expire_seconds = 0):
+        """
+        Blacklist a client from the OSDs
+
+        :param client_address: client address
+        :type client_address: str
+        :param expire_seconds: number of seconds to blacklist
+        :type expire_seconds: int
+
+        :raises: :class:`Error`
+        """
+        self.require_state("connected")
+        ret = run_in_thread(self.librados.rados_blacklist_add,
+                            (self.cluster, c_char_p(client_address),
+                             c_uint32(expire_seconds)))
+        if ret < 0:
+            raise make_ex(ret, "error blacklisting client '%s'" % client_address)
+
 class ObjectIterator(object):
     """rados.Ioctx Object iterator"""
     def __init__(self, ioctx):

src/test/librados_test_stub/LibradosTestStub.cc

@@ -678,6 +678,12 @@ AioCompletion *Rados::aio_create_completion(void *cb_arg,
   return new AioCompletion(c);
 }
 
+int Rados::blacklist_add(const std::string& client_address,
+			 uint32_t expire_seconds) {
+  TestRadosClient *impl = reinterpret_cast<TestRadosClient*>(client);
+  return impl->blacklist_add(client_address, expire_seconds);
+}
+
 int Rados::conf_parse_env(const char *env) const {
   return rados_conf_parse_env(reinterpret_cast<rados_t>(client), env);
 }
@@ -760,7 +766,7 @@ int Rados::pool_list(std::list<std::string>& v) {
   return 0;
 }
 
-int librados::Rados::pool_list2(std::list<std::pair<int64_t, std::string> >& v)
+int Rados::pool_list2(std::list<std::pair<int64_t, std::string> >& v)
 {
   TestRadosClient *impl = reinterpret_cast<TestRadosClient*>(client);
   return impl->pool_list(v);
@@ -785,6 +791,9 @@ void Rados::shutdown() {
   client = NULL;
 }
 
+void Rados::test_blacklist_self(bool set) {
+}
+
 int Rados::wait_for_latest_osdmap() {
   TestRadosClient *impl = reinterpret_cast<TestRadosClient*>(client);
   return impl->wait_for_latest_osdmap();

src/test/librados_test_stub/TestMemRadosClient.cc

@@ -119,4 +119,9 @@ int TestMemRadosClient::watch_flush() {
   return 0;
 }
 
+int TestMemRadosClient::blacklist_add(const std::string& client_address,
+				      uint32_t expire_seconds) {
+  return 0;
+}
+
 } // namespace librados

src/test/librados_test_stub/TestMemRadosClient.h

@@ -79,6 +79,9 @@ public:
   virtual int pool_reverse_lookup(int64_t id, std::string *name);
 
   virtual int watch_flush();
+
+  virtual int blacklist_add(const std::string& client_address,
+			    uint32_t expire_seconds);
 protected:
   ~TestMemRadosClient();
 

src/test/librados_test_stub/TestRadosClient.h

@@ -65,6 +65,9 @@ public:
 
   virtual int watch_flush() = 0;
 
+  virtual int blacklist_add(const std::string& client_address,
+			    uint32_t expire_seconds) = 0;
+
   TestWatchNotify &get_watch_notify() {
     return m_watch_notify;
   }

src/test/librbd/test_ImageWatcher.cc

@@ -282,15 +282,26 @@ TEST_F(TestImageWatcher, TryLockNotifyAnnounceLocked) {
 TEST_F(TestImageWatcher, TryLockWithTimedOutOwner) {
   REQUIRE_FEATURE(RBD_FEATURE_EXCLUSIVE_LOCK);
 
-  librbd::ImageCtx *ictx;
-  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+  // use new Rados connection due to blacklisting
+  librados::Rados rados;
+  ASSERT_EQ("", connect_cluster_pp(rados));
 
+  librados::IoCtx io_ctx;
+  ASSERT_EQ(0, rados.ioctx_create(_pool_name.c_str(), io_ctx));
+  librbd::ImageCtx *ictx = new librbd::ImageCtx(m_image_name.c_str(), "", NULL,
+					        io_ctx, false);
+  ASSERT_EQ(0, librbd::open_image(ictx));
   ASSERT_EQ(0, lock_image(*ictx, LOCK_EXCLUSIVE, "auto 1234"));
+  librbd::close_image(ictx);
+  io_ctx.close();
 
   // no watcher on the locked image means we can break the lock
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
   RWLock::WLocker l(ictx->owner_lock);
   ASSERT_EQ(0, ictx->image_watcher->try_lock());
   ASSERT_TRUE(ictx->image_watcher->is_lock_owner());
+
+  rados.test_blacklist_self(false);
 }
 
 TEST_F(TestImageWatcher, TryLockWithUserExclusiveLock) {

src/test/librbd/test_fixture.cc

@@ -89,7 +89,7 @@ void TestFixture::close_image(librbd::ImageCtx *ictx) {
 
 int TestFixture::lock_image(librbd::ImageCtx &ictx, ClsLockType lock_type,
 			    const std::string &cookie) {
-  int r = rados::cls::lock::lock(&m_ioctx, ictx.header_oid, RBD_LOCK_NAME,
+  int r = rados::cls::lock::lock(&ictx.md_ctx, ictx.header_oid, RBD_LOCK_NAME,
       			   lock_type, cookie, "internal", "", utime_t(),
       			   0);
   if (r == 0) {

src/test/pybind/test_rados.py

@@ -65,6 +65,7 @@ class TestRadosStateError(object):
         assert_raises(RadosStateError, rados.osd_command, 0, '', '')
         assert_raises(RadosStateError, rados.pg_command, '', '', '')
         assert_raises(RadosStateError, rados.wait_for_latest_osdmap)
+        assert_raises(RadosStateError, rados.blacklist_add, '127.0.0.1/123', 0)
 
     def test_configuring(self):
         rados = Rados(conffile='')
@@ -173,6 +174,9 @@ class TestRados(object):
         fsid = self.rados.get_fsid()
         eq(len(fsid), 36)
 
+    def test_blacklist_add(self):
+        self.rados.blacklist_add("1.2.3.4/123", 1)
+
 class TestIoctx(object):
 
     def setUp(self):