librbd: add options to enable balanced or localized reads for snapshots

Since snapshots never change, it's safe to read from replicas for them.
A common use for this would be reading from a parent snapshot shared by
many clones.

Convert LibrbdWriteback and AioRead to use the ObjectOperation api
so we can set flags. Fortunately the external wrapper holds no data,
so its lifecycle doesn't need to be managed.

Include a simple workunit that sets the flags in various combinations
and looks for their presence in the logs from 'rbd export'.

Fixes: #3064
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

qa/workunits/rbd/read-flags.sh

@@ -0,0 +1,60 @@
+#!/bin/bash -ex
+
+# create a snapshot, then export it and check that setting read flags works
+# by looking at --debug-ms output
+
+function clean_up {
+    rm -f test.log || true
+    rbd snap remove test@snap || true
+    rbd rm test || true
+}
+
+function test_read_flags {
+    local IMAGE=$1
+    local SET_BALANCED=$2
+    local SET_LOCALIZED=$3
+    local EXPECT_BALANCED=$4
+    local EXPECT_LOCALIZED=$5
+
+    local EXTRA_ARGS="--log-file test.log --debug-ms 1 --no-log-to-stderr"
+    if [ "$SET_BALANCED" = 'y' ]; then
+	EXTRA_ARGS="$EXTRA_ARGS --rbd-balance-snap-reads"
+    elif [ "$SET_LOCALIZED" = 'y' ]; then
+	EXTRA_ARGS="$EXTRA_ARGS --rbd-localize-snap-reads"
+    fi
+
+    rbd export $IMAGE - $EXTRA_ARGS > /dev/null
+    if [ "$EXPECT_BALANCED" = 'y' ]; then
+	grep -q balance_reads test.log
+    else
+	grep -L balance_reads test.log | grep -q test.log
+    fi
+    if [ "$EXPECT_LOCALIZED" = 'y' ]; then
+	grep -q localize_reads test.log
+    else
+	grep -L localize_reads test.log | grep -q test.log
+    fi
+    rm -f test.log
+
+}
+
+clean_up
+
+trap clean_up INT TERM EXIT
+
+rbd create -s 10 test
+rbd snap create test@snap
+
+# export from non snapshot with or without settings should not have flags
+test_read_flags test n n n n
+test_read_flags test y y n n
+
+# export from snapshot should have read flags in log if they are set
+test_read_flags test@snap n n n n
+test_read_flags test@snap y n y n
+test_read_flags test@snap n y n y
+
+# balanced_reads happens to take priority over localize_reads
+test_read_flags test@snap y y y n
+
+echo OK

src/common/config_opts.h

@@ -543,6 +543,8 @@ OPTION(rbd_cache_target_dirty, OPT_LONGLONG, 16<<20) // target dirty limit in by
 OPTION(rbd_cache_max_dirty_age, OPT_FLOAT, 1.0)      // seconds in cache before writeback starts
 OPTION(rbd_cache_block_writes_upfront, OPT_BOOL, false) // whether to block writes to the cache before the aio_write call completes (true), or block before the aio completion is called (false)
 OPTION(rbd_concurrent_management_ops, OPT_INT, 10) // how many operations can be in flight for a management operation like deleting or resizing an image
+OPTION(rbd_balance_snap_reads, OPT_BOOL, false)
+OPTION(rbd_localize_snap_reads, OPT_BOOL, false)
 
 OPTION(nss_db_path, OPT_STR, "") // path to nss db
 

src/librbd/AioRequest.cc

@@ -91,15 +91,16 @@ namespace librbd {
     librados::AioCompletion *rados_completion =
       librados::Rados::aio_create_completion(this, rados_req_cb, NULL);
     int r;
+    librados::ObjectReadOperation op;
+    int flags = m_ictx->get_read_flags(m_snap_id);
     if (m_sparse) {
-      r = m_ioctx->aio_sparse_read(m_oid, rados_completion, &m_ext_map,
-				   &m_read_data, m_object_len, m_object_off,
-				   m_snap_id);
+      op.sparse_read(m_object_off, m_object_len, &m_ext_map, &m_read_data,
+		     NULL);
     } else {
-      r = m_ioctx->aio_read(m_oid, rados_completion, &m_read_data,
-			    m_object_len, m_object_off,
-			    m_snap_id);
+      op.read(m_object_off, m_object_len, &m_read_data, NULL);
     }
+    r = m_ioctx->aio_operate(m_oid, rados_completion, &op, m_snap_id, flags, NULL);
+
     rados_completion->release();
     return r;
   }
@@ -116,7 +117,8 @@ namespace librbd {
 			       const ::SnapContext &snapc, librados::snap_t snap_id,
 			       Context *completion,
 			       bool hide_enoent)
-    : AioRequest(ictx, oid, object_no, object_off, len, snap_id, completion, hide_enoent),
+    : AioRequest(ictx, oid, object_no, object_off, len, snap_id, completion,
+		 hide_enoent),
       m_state(LIBRBD_AIO_WRITE_FLAT), m_snap_seq(snapc.seq.val)
   {
     m_object_image_extents = objectx;

src/librbd/AioRequest.h

@@ -66,7 +66,8 @@ namespace librbd {
 	    vector<pair<uint64_t,uint64_t> >& be,
 	    librados::snap_t snap_id, bool sparse,
 	    Context *completion)
-      : AioRequest(ictx, oid, objectno, offset, len, snap_id, completion, false),
+      : AioRequest(ictx, oid, objectno, offset, len, snap_id, completion,
+		   false),
 	m_buffer_extents(be),
 	m_tried_parent(false), m_sparse(sparse) {
     }

src/librbd/ImageCtx.cc

@@ -238,6 +238,18 @@ namespace librbd {
     delete perfcounter;
   }
 
+  int ImageCtx::get_read_flags(snap_t snap_id) {
+    int flags = librados::OPERATION_NOFLAG;
+    if (snap_id == LIBRADOS_SNAP_HEAD)
+      return flags;
+
+    if (cct->_conf->rbd_balance_snap_reads)
+      flags |= librados::OPERATION_BALANCE_READS;
+    else if (cct->_conf->rbd_localize_snap_reads)
+      flags |= librados::OPERATION_LOCALIZE_READS;
+    return flags;
+  }
+
   int ImageCtx::snap_set(string in_snap_name)
   {
     map<string, SnapInfo>::iterator it = snaps_by_name.find(in_snap_name);

src/librbd/ImageCtx.h

@@ -99,6 +99,7 @@ namespace librbd {
     void init_layout();
     void perf_start(std::string name);
     void perf_stop();
+    int get_read_flags(librados::snap_t snap_id);
     int snap_set(std::string in_snap_name);
     void snap_unset();
     librados::snap_t get_snap_id(std::string in_snap_name) const;

src/librbd/LibrbdWriteback.cc

@@ -100,8 +100,11 @@ namespace librbd {
     Context *req = new C_Request(m_ictx->cct, onfinish, &m_lock);
     librados::AioCompletion *rados_completion =
       librados::Rados::aio_create_completion(req, context_cb, NULL);
-    int r = m_ictx->data_ctx.aio_read(oid.name, rados_completion, pbl,
-				      len, off);
+    librados::ObjectReadOperation op;
+    op.read(off, len, pbl, NULL);
+    int flags = m_ictx->get_read_flags(snapid);
+    int r = m_ictx->data_ctx.aio_operate(oid.name, rados_completion, &op,
+					 snapid, flags, NULL);
     rados_completion->release();
     assert(r >= 0);
   }