From 6b7f27c0155d02c6f66d3ba52303e0f1eeb25c09 Mon Sep 17 00:00:00 2001
From: Yehuda Sadeh <yehuda@inktank.com>
Date: Wed, 4 Dec 2013 23:55:59 -0800
Subject: [PATCH] librados: use rwlock for lookup pool, cache results

Use rwlock to read cahed results, only use regular lock if
results are not cached.
Invalidate cache on new osdmap.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
---
 src/librados/RadosClient.cc | 34 ++++++++++++++++++++++++++++++++--
 src/librados/RadosClient.h  |  7 +++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc
index 06ab0d732f5..add1bb0ff14 100644
--- a/src/librados/RadosClient.cc
+++ b/src/librados/RadosClient.cc
@@ -74,7 +74,10 @@ librados::RadosClient::RadosClient(CephContext *cct_)
     messenger(NULL),
     instance_id(0),
     objecter(NULL),
+    osdmap_epoch(0),
+    pool_cache_epoch(0),
     lock("librados::RadosClient::lock"),
+    pool_cache_rwl("librados::RadosClient::pool_cache_rwl"),
     timer(cct, lock),
     refcnt(1),
     log_last_version(0), log_cb(NULL), log_cb_arg(NULL),
@@ -85,11 +88,35 @@ librados::RadosClient::RadosClient(CephContext *cct_)
 
 int64_t librados::RadosClient::lookup_pool(const char *name)
 {
-  Mutex::Locker l(lock);
+  pool_cache_rwl.get_read();
+  if (pool_cache_epoch && pool_cache_epoch == osdmap_epoch) {
+    map<string, int64_t>::iterator iter = pool_cache.find(name);
+    if (iter != pool_cache.end()) {
+      uint64_t val = iter->second;
+      pool_cache_rwl.unlock();
+      return val;
+    }
+  }
+
+  pool_cache_rwl.unlock();
+
+  lock.Lock();
+
   wait_for_osdmap();
   int64_t ret = osdmap.lookup_pg_pool_name(name);
-  if (ret < 0)
+  pool_cache_rwl.get_write();
+  lock.Unlock();
+  if (ret < 0) {
+    pool_cache_rwl.unlock();
     return -ENOENT;
+  }
+
+  if (pool_cache_epoch != osdmap_epoch) {
+    pool_cache.clear();
+    pool_cache_epoch = osdmap_epoch;
+  }
+  pool_cache[name] = ret;
+  pool_cache_rwl.unlock();
   return ret;
 }
 
@@ -340,6 +367,9 @@ bool librados::RadosClient::_dispatch(Message *m)
     break;
   case CEPH_MSG_OSD_MAP:
     objecter->handle_osd_map(static_cast<MOSDMap*>(m));
+    pool_cache_rwl.get_write();
+    osdmap_epoch = osdmap.get_epoch();
+    pool_cache_rwl.unlock();
     cond.Signal();
     break;
   case MSG_GETPOOLSTATSREPLY:
diff --git a/src/librados/RadosClient.h b/src/librados/RadosClient.h
index 87860b487d1..f9cc1f7e54c 100644
--- a/src/librados/RadosClient.h
+++ b/src/librados/RadosClient.h
@@ -16,6 +16,7 @@
 
 #include "common/Cond.h"
 #include "common/Mutex.h"
+#include "common/RWLock.h"
 #include "common/Timer.h"
 #include "include/rados/librados.h"
 #include "include/rados/librados.hpp"
@@ -62,7 +63,13 @@ private:
 
   Objecter *objecter;
 
+  map<string, int64_t> pool_cache;
+
+  epoch_t osdmap_epoch;
+  epoch_t pool_cache_epoch;
+
   Mutex lock;
+  RWLock pool_cache_rwl;
   Cond cond;
   SafeTimer timer;
   int refcnt;