Merge pull request #2652 from ceph/wip-msgr

msgr: fix remaining direct references to "new SimpleMessenger" by replacing with the factory method

Reorganize the src/msg hierarchy to push SimpleMessenger down into a subfolder to make way for organizing new in-tree implementations.

Reviewed-by: Greg Farnum <greg@inktank.com>

src/common/config_opts.h

@@ -113,6 +113,7 @@ OPTION(heartbeat_file, OPT_STR, "")
 OPTION(heartbeat_inject_failure, OPT_INT, 0)    // force an unhealthy heartbeat for N seconds
 OPTION(perf, OPT_BOOL, true)       // enable internal perf counters
 
+OPTION(ms_type, OPT_STR, "simple")   // messenger backend
 OPTION(ms_tcp_nodelay, OPT_BOOL, true)
 OPTION(ms_tcp_rcvbuf, OPT_INT, 0)
 OPTION(ms_tcp_prefetch_max_size, OPT_INT, 4096) // max prefetch size, we limit this to avoid extra memcpy

src/librados/RadosClient.cc

@@ -30,7 +30,7 @@
 
 #include "messages/MWatchNotify.h"
 #include "messages/MLog.h"
-#include "msg/SimpleMessenger.h"
+#include "msg/Messenger.h"
 
 // needed for static_cast
 #include "messages/PaxosServiceMessage.h"
@@ -209,7 +209,8 @@ int librados::RadosClient::connect()
 
   err = -ENOMEM;
   nonce = getpid() + (1000000 * (uint64_t)rados_instance.inc());
-  messenger = new SimpleMessenger(cct, entity_name_t::CLIENT(-1), "radosclient", nonce);
+  messenger = Messenger::create(cct, entity_name_t::CLIENT(-1),
+				"radosclient", nonce);
   if (!messenger)
     goto out;
 

src/librados/RadosClient.h

@@ -32,7 +32,7 @@ struct md_config_t;
 class Message;
 class MWatchNotify;
 class MLog;
-class SimpleMessenger;
+class Messenger;
 
 class librados::RadosClient : public Dispatcher
 {
@@ -47,7 +47,7 @@ private:
   } state;
 
   MonClient monclient;
-  SimpleMessenger *messenger;
+  Messenger *messenger;
 
   uint64_t instance_id;
 

src/mon/MonClient.cc

@@ -12,7 +12,7 @@
  * 
  */
 
-#include "msg/SimpleMessenger.h"
+#include "msg/Messenger.h"
 #include "messages/MMonGetMap.h"
 #include "messages/MMonGetVersion.h"
 #include "messages/MMonGetVersionReply.h"
@@ -114,11 +114,11 @@ int MonClient::get_monmap_privately()
   Mutex::Locker l(monc_lock);
 
   bool temp_msgr = false;
-  SimpleMessenger* smessenger = NULL;
+  Messenger* smessenger = NULL;
   if (!messenger) {
-    messenger = smessenger = new SimpleMessenger(cct,
-                                                 entity_name_t::CLIENT(-1),
-                                                 "temp_mon_client", getpid());
+    messenger = smessenger = Messenger::create(cct,
+					       entity_name_t::CLIENT(-1),
+					       "temp_mon_client", getpid());
     messenger->add_dispatcher_head(this);
     smessenger->start();
     temp_msgr = true;
@@ -218,9 +218,9 @@ int MonClient::ping_monitor(const string &mon_id, string *result_reply)
 
   MonClientPinger *pinger = new MonClientPinger(cct, result_reply);
 
-  Messenger *smsgr = new SimpleMessenger(cct,
-                                         entity_name_t::CLIENT(-1),
-                                         "temp_ping_client", getpid());
+  Messenger *smsgr = Messenger::create(cct,
+				       entity_name_t::CLIENT(-1),
+				       "temp_ping_client", getpid());
   smsgr->add_dispatcher_head(pinger);
   smsgr->start();
 

src/msg/Makefile.am

@@ -1,24 +1,32 @@
 libmsg_la_SOURCES = \
-	msg/Accepter.cc \
-	msg/DispatchQueue.cc \
 	msg/Message.cc \
 	msg/Messenger.cc \
-	msg/Pipe.cc \
-	msg/PipeConnection.cc \
-	msg/SimpleMessenger.cc \
 	msg/msg_types.cc
 
 noinst_HEADERS += \
-	msg/Accepter.h \
 	msg/Connection.h \
-	msg/DispatchQueue.h \
 	msg/Dispatcher.h \
 	msg/Message.h \
 	msg/Messenger.h \
-	msg/Pipe.h \
-	msg/PipeConnection.h \
-	msg/SimpleMessenger.h \
 	msg/SimplePolicyMessenger.h \
 	msg/msg_types.h
 
+# simple
+libmsg_la_SOURCES += \
+	msg/simple/Accepter.cc \
+	msg/simple/DispatchQueue.cc \
+	msg/simple/Pipe.cc \
+	msg/simple/PipeConnection.cc \
+	msg/simple/SimpleMessenger.cc
+
+noinst_HEADERS += \
+	msg/simple/Accepter.h \
+	msg/simple/DispatchQueue.h \
+	msg/simple/Pipe.h \
+	msg/simple/PipeConnection.h \
+	msg/simple/SimpleMessenger.h
+
+
 noinst_LTLIBRARIES += libmsg.la
+
+

src/msg/Message.cc

@@ -14,7 +14,7 @@ using namespace std;
 #include "global/global_context.h"
 
 #include "Message.h"
-#include "Pipe.h"
+
 #include "messages/MPGStats.h"
 
 #include "messages/MGenericMessage.h"

src/msg/Messenger.cc

@@ -2,12 +2,15 @@
 #include "include/types.h"
 #include "Messenger.h"
 
-#include "SimpleMessenger.h"
+#include "msg/simple/SimpleMessenger.h"
 
 Messenger *Messenger::create(CephContext *cct,
 			     entity_name_t name,
 			     string lname,
 			     uint64_t nonce)
 {
-  return new SimpleMessenger(cct, name, lname, nonce);
+  if (cct->_conf->ms_type == "simple")
+    return new SimpleMessenger(cct, name, lname, nonce);
+  lderr(cct) << "unrecognized ms_type '" << cct->_conf->ms_type << "'" << dendl;
+  return NULL;
 }

src/msg/simple/Accepter.cc

@@ -18,11 +18,11 @@
 #include <limits.h>
 #include <poll.h>
 
-#include "Accepter.h"
-#include "SimpleMessenger.h"
+#include "msg/Message.h"
 
-#include "Message.h"
+#include "Accepter.h"
 #include "Pipe.h"
+#include "SimpleMessenger.h"
 
 #include "common/debug.h"
 #include "common/errno.h"

src/msg/simple/Pipe.cc

@@ -18,7 +18,7 @@
 #include <limits.h>
 #include <poll.h>
 
-#include "Message.h"
+#include "msg/Message.h"
 #include "Pipe.h"
 #include "SimpleMessenger.h"
 

src/msg/simple/Pipe.h

@@ -16,10 +16,10 @@
 #define CEPH_MSGR_PIPE_H
 
 #include "include/memory.h"
-
-#include "msg_types.h"
-#include "Messenger.h"
 #include "auth/AuthSessionHandler.h"
+
+#include "msg/msg_types.h"
+#include "msg/Messenger.h"
 #include "PipeConnection.h"
 
 

src/msg/simple/PipeConnection.cc

@@ -12,7 +12,7 @@
  *
  */
 
-#include "Message.h"
+#include "msg/Message.h"
 #include "Pipe.h"
 #include "SimpleMessenger.h"
 #include "PipeConnection.h"

src/msg/simple/PipeConnection.h

@@ -15,7 +15,8 @@
 #ifndef CEPH_MSG_PIPECONNECTION_H
 #define CEPH_MSG_PIPECONNECTION_H
 
-#include "Connection.h"
+#include "msg/Connection.h"
+
 class Pipe;
 
 struct PipeConnection : public Connection {

src/msg/simple/SimpleMessenger.h

@@ -26,18 +26,18 @@ using namespace std;
 
 #include "common/Mutex.h"
 #include "include/atomic.h"
+#include "include/Spinlock.h"
 #include "common/Cond.h"
 #include "common/Thread.h"
 #include "common/Throttle.h"
 
-#include "SimplePolicyMessenger.h"
-#include "Message.h"
+#include "msg/SimplePolicyMessenger.h"
+#include "msg/Message.h"
 #include "include/assert.h"
-#include "DispatchQueue.h"
 
+#include "DispatchQueue.h"
 #include "Pipe.h"
 #include "Accepter.h"
-#include "include/Spinlock.h"
 
 /*
  * This class handles transmission and reception of messages. Generally