From ffda64119fc6165cebffd0b26619ad47b13daf51 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Mon, 24 Jan 2022 08:43:41 +0000 Subject: [PATCH] osd/scrub: create a separate chunk size conf for shallow scrubs Using the existing common default chunk size for scrubs that are not deep scrubs is wasteful: a high ratio of inter-OSD messages per chunk, while the actual OSD work per chunk is minimal. Signed-off-by: Ronen Friedman --- qa/standalone/scrub/osd-scrub-snaps.sh | 18 +++++- qa/standalone/scrub/scrub-helpers.sh | 1 + src/common/options/osd.yaml.in | 27 +++++++- src/osd/scrubber/pg_scrubber.cc | 64 ++++++++++++++----- .../osd-pg-scrub-modal.options.ts | 4 +- 5 files changed, 94 insertions(+), 20 deletions(-) diff --git a/qa/standalone/scrub/osd-scrub-snaps.sh b/qa/standalone/scrub/osd-scrub-snaps.sh index 975beb77583..c543b48a19c 100755 --- a/qa/standalone/scrub/osd-scrub-snaps.sh +++ b/qa/standalone/scrub/osd-scrub-snaps.sh @@ -207,9 +207,19 @@ function TEST_scrub_snaps() { do activate_osd $dir $osd || return 1 done + ceph tell osd.* config set osd_shallow_scrub_chunk_max 25 + ceph tell osd.* config set osd_shallow_scrub_chunk_min 5 + ceph tell osd.* config set osd_pg_stat_report_interval_max 1 + wait_for_clean || return 1 + ceph tell osd.* config get osd_shallow_scrub_chunk_max + ceph tell osd.* config get osd_shallow_scrub_chunk_min + ceph tell osd.* config get osd_pg_stat_report_interval_max + ceph tell osd.* config get osd_scrub_chunk_max + ceph tell osd.* config get osd_scrub_chunk_min + local pgid="${poolid}.0" if ! pg_scrub "$pgid" ; then return 1 @@ -759,6 +769,10 @@ function _scrub_snaps_multi() { activate_osd $dir $osd || return 1 done + ceph tell osd.* config set osd_shallow_scrub_chunk_max 3 + ceph tell osd.* config set osd_shallow_scrub_chunk_min 3 + ceph tell osd.* config set osd_scrub_chunk_min 3 + ceph tell osd.* config set osd_pg_stat_report_interval_max 1 wait_for_clean || return 1 local pgid="${poolid}.0" @@ -1149,7 +1163,7 @@ fi function TEST_scrub_snaps_replica() { local dir=$1 ORIG_ARGS=$CEPH_ARGS - CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3" + CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max=1" _scrub_snaps_multi $dir replica err=$? CEPH_ARGS=$ORIG_ARGS @@ -1159,7 +1173,7 @@ function TEST_scrub_snaps_replica() { function TEST_scrub_snaps_primary() { local dir=$1 ORIG_ARGS=$CEPH_ARGS - CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3" + CEPH_ARGS+=" --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=20 --osd_shallow_scrub_chunk_min=3 --osd_shallow_scrub_chunk_max=3 --osd_pg_stat_report_interval_max=1" _scrub_snaps_multi $dir primary err=$? CEPH_ARGS=$ORIG_ARGS diff --git a/qa/standalone/scrub/scrub-helpers.sh b/qa/standalone/scrub/scrub-helpers.sh index 38677cf7566..6816d71de32 100644 --- a/qa/standalone/scrub/scrub-helpers.sh +++ b/qa/standalone/scrub/scrub-helpers.sh @@ -239,6 +239,7 @@ function standard_scrub_cluster() { --osd_scrub_interval_randomize_ratio=0 \ --osd_scrub_backoff_ratio=0.0 \ --osd_pool_default_pg_autoscale_mode=off \ + --osd_pg_stat_report_interval_max=1 \ $extra_pars" for osd in $(seq 0 $(expr $OSDS - 1)) diff --git a/src/common/options/osd.yaml.in b/src/common/options/osd.yaml.in index 84474c34102..a1c01374a1e 100644 --- a/src/common/options/osd.yaml.in +++ b/src/common/options/osd.yaml.in @@ -333,7 +333,7 @@ options: - name: osd_scrub_chunk_min type: int level: advanced - desc: Minimum number of objects to scrub in a single chunk + desc: Minimum number of objects to deep-scrub in a single chunk fmt_desc: The minimal number of object store chunks to scrub during single operation. Ceph blocks writes to single chunk during scrub. default: 5 @@ -343,12 +343,35 @@ options: - name: osd_scrub_chunk_max type: int level: advanced - desc: Maximum number of objects to scrub in a single chunk + desc: Maximum number of objects to deep-scrub in a single chunk fmt_desc: The maximum number of object store chunks to scrub during single operation. default: 25 see_also: - osd_scrub_chunk_min with_legacy: true +- name: osd_shallow_scrub_chunk_min + type: int + level: advanced + desc: Minimum number of objects to scrub in a single chunk + fmt_desc: The minimum number of object store chunks to scrub during single operation. + Not applicable to deep scrubs. + Ceph blocks writes to single chunk during scrub. + default: 50 + see_also: + - osd_shallow_scrub_chunk_max + - osd_scrub_chunk_min + with_legacy: true +- name: osd_shallow_scrub_chunk_max + type: int + level: advanced + desc: Maximum number of objects to scrub in a single chunk + fmt_desc: The maximum number of object store chunks to scrub during single operation. + Not applicable to deep scrubs. + default: 100 + see_also: + - osd_shallow_scrub_chunk_min + - osd_scrub_chunk_max + with_legacy: true # sleep between [deep]scrub ops - name: osd_scrub_sleep type: float diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index fc5a7ec9af2..42518e88cee 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -635,6 +635,31 @@ void PgScrubber::on_applied_when_primary(const eversion_t& applied_version) } } + +namespace { + +/** + * an aux function to be used in select_range() below, to + * select the correct chunk size based on the type of scrub + */ +int size_from_conf( + bool is_deep, + const ceph::common::ConfigProxy& conf, + std::string_view deep_opt, + std::string_view shallow_opt) +{ + if (!is_deep) { + auto sz = conf.get_val(shallow_opt); + if (sz != 0) { + // assuming '0' means that no distinction was yet configured between + // deep and shallow scrubbing + return static_cast(sz); + } + } + return static_cast(conf.get_val(deep_opt)); +} +} // anonymous namespace + /* * The selected range is set directly into 'm_start' and 'm_end' * setting: @@ -663,27 +688,36 @@ bool PgScrubber::select_range() * left end of the range if we are a tier because they may legitimately * not exist (see _scrub). */ - int min_idx = static_cast( - std::max(3, - m_pg->get_cct()->_conf->osd_scrub_chunk_min / - (int)preemption_data.chunk_divisor())); - int max_idx = static_cast( - std::max(min_idx, - m_pg->get_cct()->_conf->osd_scrub_chunk_max / - (int)preemption_data.chunk_divisor())); + const auto& conf = m_pg->get_cct()->_conf; + dout(20) << fmt::format( + "{} {} mins: {}d {}s, max: {}d {}s", __func__, + (m_is_deep ? "D" : "S"), + conf.get_val("osd_scrub_chunk_min"), + conf.get_val("osd_shallow_scrub_chunk_min"), + conf.get_val("osd_scrub_chunk_max"), + conf.get_val("osd_shallow_scrub_chunk_max")) + << dendl; - dout(10) << __func__ << " Min: " << min_idx << " Max: " << max_idx - << " Div: " << preemption_data.chunk_divisor() << dendl; + const int min_from_conf = size_from_conf( + m_is_deep, conf, "osd_scrub_chunk_min", "osd_shallow_scrub_chunk_min"); + const int max_from_conf = size_from_conf( + m_is_deep, conf, "osd_scrub_chunk_max", "osd_shallow_scrub_chunk_max"); + + const int divisor = static_cast(preemption_data.chunk_divisor()); + const int min_chunk_sz = std::max(3, min_from_conf / divisor); + const int max_chunk_sz = std::max(min_chunk_sz, max_from_conf / divisor); + + dout(10) << fmt::format( + "{}: Min: {} Max: {} Div: {}", __func__, min_chunk_sz, + max_chunk_sz, divisor) + << dendl; hobject_t start = m_start; hobject_t candidate_end; std::vector objects; - int ret = m_pg->get_pgbackend()->objects_list_partial(start, - min_idx, - max_idx, - &objects, - &candidate_end); + int ret = m_pg->get_pgbackend()->objects_list_partial( + start, min_chunk_sz, max_chunk_sz, &objects, &candidate_end); ceph_assert(ret >= 0); if (!objects.empty()) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-pg-scrub-modal/osd-pg-scrub-modal.options.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-pg-scrub-modal/osd-pg-scrub-modal.options.ts index 424fbc47960..48caddd3883 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-pg-scrub-modal/osd-pg-scrub-modal.options.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-pg-scrub-modal/osd-pg-scrub-modal.options.ts @@ -31,6 +31,8 @@ export class OsdPgScrubModalOptions { 'osd_scrub_interval_randomize_ratio', 'osd_scrub_invalid_stats', 'osd_scrub_load_threshold', - 'osd_scrub_max_preemptions' + 'osd_scrub_max_preemptions', + 'osd_shallow_scrub_chunk_max', + 'osd_shallow_scrub_chunk_min' ]; }