Support for READFORWARD in the caching tier

READFORWARD caching policy uses writeback for writes
and redirects reads to the back end starage tier.

Signed-off-by: Luis Pabón <lpabon@redhat.com>

src/mon/MonCommands.h

@@ -613,7 +613,7 @@ COMMAND("osd tier remove " \
 	"osd", "rw", "cli,rest")
 COMMAND("osd tier cache-mode " \
 	"name=pool,type=CephPoolname " \
-	"name=mode,type=CephChoices,strings=none|writeback|forward|readonly", \
+	"name=mode,type=CephChoices,strings=none|writeback|forward|readonly|readforward", \
 	"specify the caching mode for cache tier <pool>", "osd", "rw", "cli,rest")
 COMMAND("osd tier set-overlay " \
 	"name=pool,type=CephPoolname " \

src/mon/OSDMonitor.cc

@@ -5315,12 +5315,14 @@ done:
      *  forward:    Forward all reads and writes to base pool
      *  writeback:  Cache writes, promote reads from base pool
      *  readonly:   Forward writes to base pool
+     *  readforward: Writes are in writeback mode, Reads and in forward mode
      *
      * Hence, these are the allowed transitions:
      *
      *  none -> any
-     *  forward -> writeback || any IF num_objects_dirty == 0
-     *  writeback -> forward
+     *  forward -> readforward || writeback || any IF num_objects_dirty == 0
+     *  readforward -> forward || writeback || any IF num_objects_dirty == 0
+     *  writeback -> readforward || forward
      *  readonly -> any
      */
 
@@ -5329,17 +5331,25 @@ done:
     // whatever mode is on the pending state.
 
     if (p->cache_mode == pg_pool_t::CACHEMODE_WRITEBACK &&
-        mode != pg_pool_t::CACHEMODE_FORWARD) {
+        (mode != pg_pool_t::CACHEMODE_FORWARD &&
+	  mode != pg_pool_t::CACHEMODE_READFORWARD)) {
       ss << "unable to set cache-mode '" << pg_pool_t::get_cache_mode_name(mode)
          << "' on a '" << pg_pool_t::get_cache_mode_name(p->cache_mode)
          << "' pool; only '"
          << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_FORWARD)
+	 << "','"
+         << pg_pool_t::get_cache_mode_name(pg_pool_t::CACHEMODE_READFORWARD)
         << "' allowed.";
       err = -EINVAL;
       goto reply;
     }
-    if (p->cache_mode == pg_pool_t::CACHEMODE_FORWARD &&
-               mode != pg_pool_t::CACHEMODE_WRITEBACK) {
+    if ((p->cache_mode == pg_pool_t::CACHEMODE_READFORWARD &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	  mode != pg_pool_t::CACHEMODE_FORWARD)) ||
+
+        (p->cache_mode == pg_pool_t::CACHEMODE_FORWARD &&
+        (mode != pg_pool_t::CACHEMODE_WRITEBACK &&
+	  mode != pg_pool_t::CACHEMODE_READFORWARD))) {
 
       const pool_stat_t& tier_stats =
         mon->pgmon()->pg_map.get_pg_pool_sum_stat(pool_id);
@@ -5352,7 +5362,6 @@ done:
         goto reply;
       }
     }
-
     // go
     pending_inc.get_new_pool(pool_id, p)->cache_mode = mode;
     ss << "set cache-mode for pool '" << poolstr

src/osd/ReplicatedPG.cc

@@ -1624,6 +1624,33 @@ bool ReplicatedPG::maybe_handle_cache(OpRequestRef op,
     // crap, there was a failure of some kind
     return false;
 
+  case pg_pool_t::CACHEMODE_READFORWARD:
+    if (obc.get() && obc->obs.exists) {
+      return false;
+    }
+
+    // Do writeback to the cache tier for writes
+    if (op->may_write()) {
+      if (agent_state &&
+	  agent_state->evict_mode == TierAgentState::EVICT_MODE_FULL) {
+	dout(20) << __func__ << " cache pool full, waiting" << dendl;
+	waiting_for_cache_not_full.push_back(op);
+	return true;
+      }
+      if (!must_promote && can_skip_promote(op, obc)) {
+	return false;
+      }
+      promote_object(op, obc, missing_oid);
+      return true;
+    }
+
+    // If it is a read, we can read, we need to forward it
+    if (must_promote)
+      promote_object(op, obc, missing_oid);
+    else
+      do_cache_redirect(op, obc);
+    return true;
+
   default:
     assert(0 == "unrecognized cache_mode");
   }
@@ -5899,6 +5926,7 @@ void ReplicatedPG::finish_promote(int r, OpRequestRef op,
   if (r == -ENOENT &&
       soid.snap == CEPH_NOSNAP &&
       (pool.info.cache_mode == pg_pool_t::CACHEMODE_WRITEBACK ||
+       pool.info.cache_mode == pg_pool_t::CACHEMODE_READFORWARD ||
        pool.info.cache_mode == pg_pool_t::CACHEMODE_READONLY)) {
     dout(10) << __func__ << " whiteout " << soid << dendl;
     whiteout = true;

src/osd/osd_types.h

@@ -846,6 +846,7 @@ struct pg_pool_t {
     CACHEMODE_WRITEBACK = 1,             ///< write to cache, flush later
     CACHEMODE_FORWARD = 2,               ///< forward if not in cache
     CACHEMODE_READONLY = 3,              ///< handle reads, forward writes [not strongly consistent]
+    CACHEMODE_READFORWARD = 4            ///< forward reads, write to cache flush later
   } cache_mode_t;
   static const char *get_cache_mode_name(cache_mode_t m) {
     switch (m) {
@@ -853,6 +854,7 @@ struct pg_pool_t {
     case CACHEMODE_WRITEBACK: return "writeback";
     case CACHEMODE_FORWARD: return "forward";
     case CACHEMODE_READONLY: return "readonly";
+    case CACHEMODE_READFORWARD: return "readforward";
     default: return "unknown";
     }
   }
@@ -865,6 +867,8 @@ struct pg_pool_t {
       return CACHEMODE_FORWARD;
     if (s == "readonly")
       return CACHEMODE_READONLY;
+    if (s == "readforward")
+      return CACHEMODE_READFORWARD;
     return (cache_mode_t)-1;
   }
   const char *get_cache_mode_name() const {

src/test/pybind/test_ceph_argparse.py

@@ -1094,7 +1094,7 @@ class TestOSD(TestArgparse):
                                                         'toomany']))
 
     def test_tier_cache_mode(self):
-        for mode in ('none', 'writeback', 'forward', 'readonly'):
+        for mode in ('none', 'writeback', 'forward', 'readonly', 'readforward'):
             self.assert_valid_command(['osd', 'tier', 'cache-mode',
                                        'poolname', mode])
         assert_equal({}, validate_command(sigdict, ['osd', 'tier',