From 054741725f1272e035366d452027648966b7ae6d Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:43:47 -0700 Subject: [PATCH 1/9] common/RefCountedObject: optionally debug Introduce a 'refs' subsys for logging. Print log ref count inc and dec for any RefCountedObject that gets a cct passed to its ctor. Signed-off-by: Sage Weil --- src/common/RefCountedObj.h | 20 ++++++++++++++------ src/common/config_opts.h | 1 + 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/common/RefCountedObj.h b/src/common/RefCountedObj.h index 042adb58780..84bd22f57b0 100644 --- a/src/common/RefCountedObj.h +++ b/src/common/RefCountedObj.h @@ -18,21 +18,29 @@ #include "common/Mutex.h" #include "common/Cond.h" #include "include/atomic.h" - +#include "common/ceph_context.h" struct RefCountedObject { atomic_t nref; - RefCountedObject() : nref(1) {} + CephContext *cct; + RefCountedObject(CephContext *c = NULL) : nref(1), cct(c) {} virtual ~RefCountedObject() {} RefCountedObject *get() { - //generic_dout(0) << "RefCountedObject::get " << this << " " << nref.read() << " -> " << (nref.read() + 1) << dendl; - nref.inc(); + int v = nref.inc(); + if (cct) + lsubdout(cct, refs, 1) << "RefCountedObject::get " << this << " " + << (v - 1) << " -> " << v + << dendl; return this; } void put() { - //generic_dout(0) << "RefCountedObject::put " << this << " " << nref.read() << " -> " << (nref.read() - 1) << dendl; - if (nref.dec() == 0) + int v = nref.dec(); + if (cct) + lsubdout(cct, refs, 1) << "RefCountedObject::put " << this << " " + << (v + 1) << " -> " << v + << dendl; + if (v == 0) delete this; } }; diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 1c9dbacc71d..1257ae47a4a 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -95,6 +95,7 @@ SUBSYS(rgw, 1, 5) // log level for the Rados gateway SUBSYS(javaclient, 1, 5) SUBSYS(asok, 1, 5) SUBSYS(throttle, 1, 1) +SUBSYS(refs, 0, 0) OPTION(key, OPT_STR, "") OPTION(keyfile, OPT_STR, "") From ff1521fce4a77c2836aa99bebccbbb8c239e60f8 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:49:58 -0700 Subject: [PATCH 2/9] common/RefCountedObject: optionally take initial ref count to ctor Signed-off-by: Sage Weil --- src/common/RefCountedObj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/RefCountedObj.h b/src/common/RefCountedObj.h index 84bd22f57b0..ca405693081 100644 --- a/src/common/RefCountedObj.h +++ b/src/common/RefCountedObj.h @@ -23,7 +23,7 @@ struct RefCountedObject { atomic_t nref; CephContext *cct; - RefCountedObject(CephContext *c = NULL) : nref(1), cct(c) {} + RefCountedObject(CephContext *c = NULL, int n=1) : nref(n), cct(c) {} virtual ~RefCountedObject() {} RefCountedObject *get() { From 45991c055bc949f6b81f93c25e674e60fc7649e7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:51:06 -0700 Subject: [PATCH 3/9] common/RefCountedObject: assert nref == 0 in dtor Signed-off-by: Sage Weil --- src/common/RefCountedObj.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/RefCountedObj.h b/src/common/RefCountedObj.h index ca405693081..395a4368734 100644 --- a/src/common/RefCountedObj.h +++ b/src/common/RefCountedObj.h @@ -24,7 +24,9 @@ struct RefCountedObject { atomic_t nref; CephContext *cct; RefCountedObject(CephContext *c = NULL, int n=1) : nref(n), cct(c) {} - virtual ~RefCountedObject() {} + virtual ~RefCountedObject() { + assert(nref.read() == 0); + } RefCountedObject *get() { int v = nref.inc(); From fc8d1980495c7ba13edd59929e65071fcd396f70 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:44:11 -0700 Subject: [PATCH 4/9] osd: debug Session refs Signed-off-by: Sage Weil --- src/osd/OSD.cc | 6 +++--- src/osd/OSD.h | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index b396e12366e..6006165f646 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3691,7 +3691,7 @@ void OSD::ms_handle_fast_connect(Connection *con) if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON) { Session *s = static_cast(con->get_priv()); if (!s) { - s = new Session; + s = new Session(cct); con->set_priv(s->get()); s->con = con; dout(10) << " new session (outgoing)" << s << " con=" << s->con @@ -3709,7 +3709,7 @@ void OSD::ms_handle_fast_accept(Connection *con) if (con->get_peer_type() != CEPH_ENTITY_TYPE_MON) { Session *s = static_cast(con->get_priv()); if (!s) { - s = new Session(); + s = new Session(cct); con->set_priv(s->get()); s->con = con; dout(10) << "new session (incoming)" << s << " con=" << con @@ -5142,7 +5142,7 @@ bool OSD::ms_verify_authorizer(Connection *con, int peer_type, if (isvalid) { Session *s = static_cast(con->get_priv()); if (!s) { - s = new Session; + s = new Session(cct); con->set_priv(s->get()); s->con = con; dout(10) << " new session " << s << " con=" << s->con << " addr=" << s->con->get_peer_addr() << dendl; diff --git a/src/osd/OSD.h b/src/osd/OSD.h index 428f903bb61..cbac62ab6da 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1152,7 +1152,8 @@ public: Mutex received_map_lock; epoch_t received_map_epoch; // largest epoch seen in MOSDMap from here - Session() : + Session(CephContext *cct) : + RefCountedObject(cct), auid(-1), con(0), session_dispatch_lock("Session::session_dispatch_lock"), sent_epoch_lock("Session::sent_epoch_lock"), last_sent_epoch(0), From 242ee89c96848ee816241e9e7be08ea56fb26f06 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 16:43:46 -0700 Subject: [PATCH 5/9] msg: debug Connection refs Signed-off-by: Sage Weil --- src/mon/Monitor.cc | 2 +- src/msg/Message.h | 5 +++-- src/msg/Pipe.cc | 2 +- src/msg/SimpleMessenger.cc | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index f983284eb8d..8e101244795 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -2633,7 +2633,7 @@ void Monitor::handle_forward(MForward *m) dout(0) << "forward from entity with insufficient caps! " << session->caps << dendl; } else { - Connection *c = new Connection(NULL); // msgr must be null; see PaxosService::dispatch() + Connection *c = new Connection(g_ceph_context, NULL); // msgr must be null; see PaxosService::dispatch() MonSession *s = new MonSession(m->msg->get_source_inst(), c); c->set_priv(s); c->set_peer_addr(m->client.addr); diff --git a/src/msg/Message.h b/src/msg/Message.h index f3c9a4c8dca..0205eed957e 100644 --- a/src/msg/Message.h +++ b/src/msg/Message.h @@ -190,8 +190,9 @@ public: friend class boost::intrusive_ptr; public: - Connection(Messenger *m) - : lock("Connection::lock"), + Connection(CephContext *cct, Messenger *m) + : RefCountedObject(cct), + lock("Connection::lock"), msgr(m), priv(NULL), peer_type(-1), diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc index e0c5ef9dc8e..68825f8c1c8 100644 --- a/src/msg/Pipe.cc +++ b/src/msg/Pipe.cc @@ -95,7 +95,7 @@ Pipe::Pipe(SimpleMessenger *r, int st, Connection *con) connection_state = con; connection_state->reset_pipe(this); } else { - connection_state = new Connection(msgr); + connection_state = new Connection(msgr->cct, msgr); connection_state->pipe = get(); } diff --git a/src/msg/SimpleMessenger.cc b/src/msg/SimpleMessenger.cc index 4a8711f722d..3962d3bc092 100644 --- a/src/msg/SimpleMessenger.cc +++ b/src/msg/SimpleMessenger.cc @@ -52,7 +52,7 @@ SimpleMessenger::SimpleMessenger(CephContext *cct, entity_name_t name, dispatch_throttler(cct, string("msgr_dispatch_throttler-") + mname, cct->_conf->ms_dispatch_throttle_bytes), reaper_started(false), reaper_stop(false), timeout(0), - local_connection(new Connection(this)) + local_connection(new Connection(cct, this)) { ceph_spin_init(&global_seq_lock); init_local_connection(); From e6218568477c7efe10a756c261412532029cefd3 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 16:44:02 -0700 Subject: [PATCH 6/9] msg/Pipe: debug Pipe refs Signed-off-by: Sage Weil --- src/msg/Pipe.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/msg/Pipe.cc b/src/msg/Pipe.cc index 68825f8c1c8..59179e0b7fd 100644 --- a/src/msg/Pipe.cc +++ b/src/msg/Pipe.cc @@ -73,7 +73,7 @@ ostream& Pipe::_pipe_prefix(std::ostream *_dout) { */ Pipe::Pipe(SimpleMessenger *r, int st, Connection *con) - : reader_thread(this), writer_thread(this), + : RefCountedObject(r->cct), reader_thread(this), writer_thread(this), delay_thread(NULL), msgr(r), conn_id(r->dispatch_queue.get_id()), From b0da92b895d9303b46b41b9e76f4096a768a679d Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:50:33 -0700 Subject: [PATCH 7/9] msg/Message: use RefCountedObject ctor to set initial ref count Also, drop the useless assert in the dtor (the RefCountedObject dtor does the same). Signed-off-by: Sage Weil --- src/msg/Message.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/msg/Message.h b/src/msg/Message.h index 0205eed957e..191263f482f 100644 --- a/src/msg/Message.h +++ b/src/msg/Message.h @@ -191,7 +191,9 @@ public: public: Connection(CephContext *cct, Messenger *m) - : RefCountedObject(cct), + // we are managed exlusively by ConnectionRef; make it so you can + // ConnectionRef foo = new Connection; + : RefCountedObject(cct, 0), lock("Connection::lock"), msgr(m), priv(NULL), @@ -200,9 +202,6 @@ public: pipe(NULL), failed(false), rx_buffers_version(0) { - // we are managed exlusively by ConnectionRef; make it so you can - // ConnectionRef foo = new Connection; - nref.set(0); } ~Connection() { //generic_dout(0) << "~Connection " << this << dendl; @@ -372,7 +371,6 @@ public: protected: virtual ~Message() { - assert(nref.read() == 0); if (byte_throttler) byte_throttler->put(payload.length() + middle.length() + data.length()); if (msg_throttler) From c0dcf3bf47fde3365c38811d512acdb1bda7da4c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:51:30 -0700 Subject: [PATCH 8/9] common/RefCountedObject: make nref, cct private Signed-off-by: Sage Weil --- src/common/RefCountedObj.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common/RefCountedObj.h b/src/common/RefCountedObj.h index 395a4368734..f9644a791ce 100644 --- a/src/common/RefCountedObj.h +++ b/src/common/RefCountedObj.h @@ -21,8 +21,10 @@ #include "common/ceph_context.h" struct RefCountedObject { +private: atomic_t nref; CephContext *cct; +public: RefCountedObject(CephContext *c = NULL, int n=1) : nref(n), cct(c) {} virtual ~RefCountedObject() { assert(nref.read() == 0); @@ -45,6 +47,9 @@ struct RefCountedObject { if (v == 0) delete this; } + void set_cct(CephContext *c) { + cct = c; + } }; /** From c0ba58c49f6b400bcae2fa9082972894caf001d5 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 1 Jul 2014 15:47:44 -0700 Subject: [PATCH 9/9] msg: debug refs on incoming Messages Signed-off-by: Sage Weil --- src/msg/Message.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/msg/Message.cc b/src/msg/Message.cc index 6ecce716924..8aa5fa0c855 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -687,6 +687,8 @@ Message *decode_message(CephContext *cct, ceph_msg_header& header, ceph_msg_foot return 0; } + m->set_cct(cct); + // m->header.version, if non-zero, should be populated with the // newest version of the encoding the code supports. If set, check // it against compat_version.