Merge pull request #44749 from ronen-fr/wip-rf-large-chunk

osd/scrub: use separate chunk size configuration for shallow scrubs

Reviewed-by: Samuel Just <sjust@redhat.com>
Reviewed-by: Josh Durgin <jdurgin@redhat.com>
Reviewed-by: Mark Kogan <mkogan@redhat.com>

qa/standalone/scrub/osd-scrub-snaps.sh

@@ -207,9 +207,19 @@ function TEST_scrub_snaps() {
     do
       activate_osd $dir $osd || return 1
     done
+    ceph tell osd.* config set osd_shallow_scrub_chunk_max 25
+    ceph tell osd.* config set osd_shallow_scrub_chunk_min 5
+    ceph tell osd.* config set osd_pg_stat_report_interval_max 1
+
 
     wait_for_clean || return 1
 
+    ceph tell osd.* config get osd_shallow_scrub_chunk_max
+    ceph tell osd.* config get osd_shallow_scrub_chunk_min
+    ceph tell osd.* config get osd_pg_stat_report_interval_max
+    ceph tell osd.* config get osd_scrub_chunk_max
+    ceph tell osd.* config get osd_scrub_chunk_min
+
     local pgid="${poolid}.0"
     if ! pg_scrub "$pgid" ; then
         return 1
@@ -759,6 +769,10 @@ function _scrub_snaps_multi() {
       activate_osd $dir $osd || return 1
     done
 
+    ceph tell osd.* config set osd_shallow_scrub_chunk_max 3
+    ceph tell osd.* config set osd_shallow_scrub_chunk_min 3
+    ceph tell osd.* config set osd_scrub_chunk_min 3
+    ceph tell osd.* config set osd_pg_stat_report_interval_max 1
     wait_for_clean || return 1
 
     local pgid="${poolid}.0"
@@ -1149,7 +1163,7 @@ fi
 function TEST_scrub_snaps_replica() {
     local dir=$1
     ORIG_ARGS=$CEPH_ARGS
-    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3"
+    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max=1"
     _scrub_snaps_multi $dir replica
     err=$?
     CEPH_ARGS=$ORIG_ARGS
@@ -1159,7 +1173,7 @@ function TEST_scrub_snaps_replica() {
 function TEST_scrub_snaps_primary() {
     local dir=$1
     ORIG_ARGS=$CEPH_ARGS
-    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3"
+    CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max=1"
     _scrub_snaps_multi $dir primary
     err=$?
     CEPH_ARGS=$ORIG_ARGS

qa/standalone/scrub/scrub-helpers.sh

@@ -239,6 +239,7 @@ function standard_scrub_cluster() {
             --osd_scrub_interval_randomize_ratio=0 \
             --osd_scrub_backoff_ratio=0.0 \
             --osd_pool_default_pg_autoscale_mode=off \
+            --osd_pg_stat_report_interval_max=1 \
             $extra_pars"
 
     for osd in $(seq 0 $(expr $OSDS - 1))

src/common/options/osd.yaml.in

@@ -333,7 +333,7 @@ options:
 - name: osd_scrub_chunk_min
   type: int
   level: advanced
-  desc: Minimum number of objects to scrub in a single chunk
+  desc: Minimum number of objects to deep-scrub in a single chunk
   fmt_desc: The minimal number of object store chunks to scrub during single operation.
     Ceph blocks writes to single chunk during scrub.
   default: 5
@@ -343,12 +343,35 @@ options:
 - name: osd_scrub_chunk_max
   type: int
   level: advanced
-  desc: Maximum number of objects to scrub in a single chunk
+  desc: Maximum number of objects to deep-scrub in a single chunk
   fmt_desc: The maximum number of object store chunks to scrub during single operation.
   default: 25
   see_also:
   - osd_scrub_chunk_min
   with_legacy: true
+- name: osd_shallow_scrub_chunk_min
+  type: int
+  level: advanced
+  desc: Minimum number of objects to scrub in a single chunk
+  fmt_desc: The minimum number of object store chunks to scrub during single operation.
+    Not applicable to deep scrubs.
+    Ceph blocks writes to single chunk during scrub.
+  default: 50
+  see_also:
+  - osd_shallow_scrub_chunk_max
+  - osd_scrub_chunk_min
+  with_legacy: true
+- name: osd_shallow_scrub_chunk_max
+  type: int
+  level: advanced
+  desc: Maximum number of objects to scrub in a single chunk
+  fmt_desc: The maximum number of object store chunks to scrub during single operation.
+    Not applicable to deep scrubs.
+  default: 100
+  see_also:
+  - osd_shallow_scrub_chunk_min
+  - osd_scrub_chunk_max
+  with_legacy: true
 # sleep between [deep]scrub ops
 - name: osd_scrub_sleep
   type: float

src/osd/scrubber/pg_scrubber.cc

@@ -635,6 +635,31 @@ void PgScrubber::on_applied_when_primary(const eversion_t& applied_version)
   }
 }
 
+
+namespace {
+
+/**
+ * an aux function to be used in select_range() below, to
+ * select the correct chunk size based on the type of scrub
+ */
+int size_from_conf(
+    bool is_deep,
+    const ceph::common::ConfigProxy& conf,
+    std::string_view deep_opt,
+    std::string_view shallow_opt)
+{
+  if (!is_deep) {
+    auto sz = conf.get_val<int64_t>(shallow_opt);
+    if (sz != 0) {
+      // assuming '0' means that no distinction was yet configured between
+      // deep and shallow scrubbing
+      return static_cast<int>(sz);
+    }
+  }
+  return static_cast<int>(conf.get_val<int64_t>(deep_opt));
+}
+}  // anonymous namespace
+
 /*
  * The selected range is set directly into 'm_start' and 'm_end'
  * setting:
@@ -663,27 +688,36 @@ bool PgScrubber::select_range()
    * left end of the range if we are a tier because they may legitimately
    * not exist (see _scrub).
    */
-  int min_idx = static_cast<int>(
-    std::max<int64_t>(3,
-		      m_pg->get_cct()->_conf->osd_scrub_chunk_min /
-			(int)preemption_data.chunk_divisor()));
 
-  int max_idx = static_cast<int>(
-    std::max<int64_t>(min_idx,
-		      m_pg->get_cct()->_conf->osd_scrub_chunk_max /
-			(int)preemption_data.chunk_divisor()));
+  const auto& conf = m_pg->get_cct()->_conf;
+  dout(20) << fmt::format(
+		  "{} {} mins: {}d {}s, max: {}d {}s", __func__,
+		  (m_is_deep ? "D" : "S"),
+		  conf.get_val<int64_t>("osd_scrub_chunk_min"),
+		  conf.get_val<int64_t>("osd_shallow_scrub_chunk_min"),
+		  conf.get_val<int64_t>("osd_scrub_chunk_max"),
+		  conf.get_val<int64_t>("osd_shallow_scrub_chunk_max"))
+	   << dendl;
 
-  dout(10) << __func__ << " Min: " << min_idx << " Max: " << max_idx
-	   << " Div: " << preemption_data.chunk_divisor() << dendl;
+  const int min_from_conf = size_from_conf(
+      m_is_deep, conf, "osd_scrub_chunk_min", "osd_shallow_scrub_chunk_min");
+  const int max_from_conf = size_from_conf(
+      m_is_deep, conf, "osd_scrub_chunk_max", "osd_shallow_scrub_chunk_max");
+
+  const int divisor = static_cast<int>(preemption_data.chunk_divisor());
+  const int min_chunk_sz = std::max(3, min_from_conf / divisor);
+  const int max_chunk_sz = std::max(min_chunk_sz, max_from_conf / divisor);
+
+  dout(10) << fmt::format(
+		  "{}: Min: {} Max: {} Div: {}", __func__, min_chunk_sz,
+		  max_chunk_sz, divisor)
+	   << dendl;
 
   hobject_t start = m_start;
   hobject_t candidate_end;
   std::vector<hobject_t> objects;
-  int ret = m_pg->get_pgbackend()->objects_list_partial(start,
-							min_idx,
-							max_idx,
-							&objects,
-							&candidate_end);
+  int ret = m_pg->get_pgbackend()->objects_list_partial(
+      start, min_chunk_sz, max_chunk_sz, &objects, &candidate_end);
   ceph_assert(ret >= 0);
 
   if (!objects.empty()) {

src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-pg-scrub-modal/osd-pg-scrub-modal.options.ts

@@ -31,6 +31,8 @@ export class OsdPgScrubModalOptions {
     'osd_scrub_interval_randomize_ratio',
     'osd_scrub_invalid_stats',
     'osd_scrub_load_threshold',
-    'osd_scrub_max_preemptions'
+    'osd_scrub_max_preemptions',
+    'osd_shallow_scrub_chunk_max',
+    'osd_shallow_scrub_chunk_min'
   ];
 }