Merge pull request #33493 from zhengchengyao/read

librbd: improved support for balanced and localized reads

Reviewed-by: Jason Dillaman <dillaman@redhat.com>

doc/rbd/rbd-config-ref.rst

@@ -15,6 +15,14 @@ Generic IO Settings
 :Default: ``none``
 :Values: ``none``, ``compressible``, ``incompressible``
 
+``rbd read from replica policy``
+
+:Description: policy for determining which OSD will receive read operations. If set to `default`, the primary OSD will always be used for read operations. If set to `balance`, read operations will be sent to a randomly selected OSD within the replica set. If set to `localize`, read operations will be sent to the closest OSD as determined by the CRUSH map. Note: this feature requires the cluster to be configured with a minimum compatible OSD release of Octopus.
+:Type: Enum
+:Required: No
+:Default: ``default``
+:Values: ``default``, ``balance``, ``localize``
+
 Cache Settings
 =======================
 

qa/suites/rbd/qemu/features/readbalance.yaml

@@ -0,0 +1,11 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rbd read from replica policy: balance
+
+tasks:
+- exec:
+    osd.0:
+      - ceph osd require-osd-release octopus
+      - ceph osd set-require-min-compat-client octopus

src/common/options.cc

@@ -7180,6 +7180,12 @@ static std::vector<Option> get_rbd_options() {
     .set_description("Compression hint to send to the OSDs during writes")
     .set_flag(Option::FLAG_RUNTIME),
 
+    Option("rbd_read_from_replica_policy", Option::TYPE_STR, Option::LEVEL_BASIC)
+    .set_enum_allowed({"default", "balance", "localize"})
+    .set_default("default")
+    .set_description("Read replica policy send to the OSDS during reads")
+    .set_flag(Option::FLAG_RUNTIME),
+
     Option("rbd_tracing", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(false)
     .set_description("true if LTTng-UST tracepoints should be enabled"),

src/librbd/ImageCtx.cc

@@ -309,7 +309,11 @@ public:
   }
 
   int ImageCtx::get_read_flags(snap_t snap_id) {
-    int flags = librados::OPERATION_NOFLAG | extra_read_flags;
+    int flags = librados::OPERATION_NOFLAG | read_flags;
+    if (flags != 0)
+      return flags;
+
+    flags = librados::OPERATION_NOFLAG | extra_read_flags;
     if (snap_id == LIBRADOS_SNAP_HEAD)
       return flags;
 
@@ -795,6 +799,19 @@ public:
       alloc_hint_flags |= librados::ALLOC_HINT_FLAG_INCOMPRESSIBLE;
     }
 
+    librados::Rados rados(md_ctx);
+    int8_t require_osd_release;
+    int r = rados.get_min_compatible_osd(&require_osd_release);
+    if (r == 0 && require_osd_release >= CEPH_RELEASE_OCTOPUS) {
+      read_flags = 0;
+      auto read_policy = config.get_val<std::string>("rbd_read_from_replica_policy");
+      if (read_policy == "balance") {
+        read_flags |= CEPH_OSD_FLAG_BALANCE_READS;
+      } else if (read_policy == "localize") {
+        read_flags |= CEPH_OSD_FLAG_LOCALIZE_READS;
+      }
+    }
+
     io_work_queue->apply_qos_schedule_tick_min(
       config.get_val<uint64_t>("rbd_qos_schedule_tick_min"));
 

src/librbd/ImageCtx.h

@@ -207,6 +207,7 @@ namespace librbd {
     bool clone_copy_on_read;
     bool enable_alloc_hint;
     uint32_t alloc_hint_flags = 0U;
+    uint32_t read_flags = 0U;
     uint32_t discard_granularity_bytes = 0;
     bool blkin_trace_all;
     uint64_t mirroring_replay_delay;

src/test/librbd/mock/MockImageCtx.h

@@ -98,6 +98,7 @@ struct MockImageCtx {
       blkin_trace_all(image_ctx.blkin_trace_all),
       enable_alloc_hint(image_ctx.enable_alloc_hint),
       alloc_hint_flags(image_ctx.alloc_hint_flags),
+      read_flags(image_ctx.read_flags),
       ignore_migrating(image_ctx.ignore_migrating),
       enable_sparse_copyup(image_ctx.enable_sparse_copyup),
       mtime_update_interval(image_ctx.mtime_update_interval),
@@ -305,6 +306,7 @@ struct MockImageCtx {
   bool blkin_trace_all;
   bool enable_alloc_hint;
   uint32_t alloc_hint_flags;
+  uint32_t read_flags;
   bool ignore_migrating;
   bool enable_sparse_copyup;
   uint64_t mtime_update_interval;

src/test/librbd/test_librbd.cc

@@ -1948,6 +1948,7 @@ TEST_F(TestLibRBD, TestIO)
   uint64_t size = 2 << 20;  
 
   ASSERT_EQ(0, create_image(ioctx, name.c_str(), size, &order));
+  ASSERT_EQ(0, rados_conf_set(_cluster, "rbd_read_from_replica_policy", "balance"));
   ASSERT_EQ(0, rbd_open(ioctx, name.c_str(), &image, NULL));
   
   char test_data[TEST_IO_SIZE + 1];