From fffa5e201a5aa8db5454043d6c32d232cae4abd2 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Wed, 4 Feb 2015 22:32:57 +0800 Subject: [PATCH 1/2] osd,mon: add norebalance osdmap flag Fixes: #10559 Signed-off-by: Kefu Chai --- doc/man/8/ceph.rst | 8 ++++---- qa/workunits/cephtool/test.sh | 2 +- src/include/rados.h | 1 + src/mon/MonCommands.h | 4 ++-- src/mon/OSDMonitor.cc | 5 +++++ src/osd/OSDMap.cc | 2 ++ src/osd/PG.cc | 3 ++- src/osd/ReplicatedPG.cc | 4 ++++ 8 files changed, 21 insertions(+), 8 deletions(-) diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst index 223a998db19..5a3121feccb 100644 --- a/doc/man/8/ceph.rst +++ b/doc/man/8/ceph.rst @@ -820,8 +820,8 @@ Subcommand ``set`` sets . Usage:: - ceph osd set pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub| - nodeep-scrub|notieragent + ceph osd set pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover| + noscrub|nodeep-scrub|notieragent Subcommand ``setcrushmap`` sets crush map from input file. @@ -906,8 +906,8 @@ Subcommand ``unset`` unsets . Usage:: - osd unset pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub| - nodeep-scrub|notieragent + osd unset pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover| + noscrub|nodeep-scrub|notieragent pg diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index b333b4f5e19..6251af1af27 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -879,7 +879,7 @@ function test_mon_osd() ceph osd deep-scrub 0 ceph osd repair 0 - for f in noup nodown noin noout noscrub nodeep-scrub nobackfill norecover notieragent full + for f in noup nodown noin noout noscrub nodeep-scrub nobackfill norebalance norecover notieragent full do ceph osd set $f ceph osd unset $f diff --git a/src/include/rados.h b/src/include/rados.h index da166cd1604..3691a2ceadb 100644 --- a/src/include/rados.h +++ b/src/include/rados.h @@ -142,6 +142,7 @@ extern const char *ceph_osd_state_name(int s); #define CEPH_OSDMAP_NOSCRUB (1<<11) /* block periodic scrub */ #define CEPH_OSDMAP_NODEEP_SCRUB (1<<12) /* block periodic deep-scrub */ #define CEPH_OSDMAP_NOTIERAGENT (1<<13) /* disable tiering agent */ +#define CEPH_OSDMAP_NOREBALANCE (1<<14) /* block osd backfill unless pg is degraded */ /* * The error code to return when an OSD can't handle a write diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index e11bc734b36..b671e063fea 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -518,10 +518,10 @@ COMMAND("osd erasure-code-profile ls", \ "list all erasure code profiles", \ "osd", "r", "cli,rest") COMMAND("osd set " \ - "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub|notieragent", \ + "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent", \ "set ", "osd", "rw", "cli,rest") COMMAND("osd unset " \ - "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norecover|noscrub|nodeep-scrub|notieragent", \ + "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent", \ "unset ", "osd", "rw", "cli,rest") COMMAND("osd cluster_snap", "take cluster snapshot (disabled)", \ "osd", "r", "") diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 9b1f9801a16..3c118530f94 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -2473,6 +2473,7 @@ void OSDMonitor::get_health(list >& summary, CEPH_OSDMAP_NOIN | CEPH_OSDMAP_NOOUT | CEPH_OSDMAP_NOBACKFILL | + CEPH_OSDMAP_NOREBALANCE | CEPH_OSDMAP_NORECOVER | CEPH_OSDMAP_NOSCRUB | CEPH_OSDMAP_NODEEP_SCRUB | @@ -5279,6 +5280,8 @@ bool OSDMonitor::prepare_command_impl(MMonCommand *m, return prepare_set_flag(m, CEPH_OSDMAP_NOIN); else if (key == "nobackfill") return prepare_set_flag(m, CEPH_OSDMAP_NOBACKFILL); + else if (key == "norebalance") + return prepare_set_flag(m, CEPH_OSDMAP_NOREBALANCE); else if (key == "norecover") return prepare_set_flag(m, CEPH_OSDMAP_NORECOVER); else if (key == "noscrub") @@ -5309,6 +5312,8 @@ bool OSDMonitor::prepare_command_impl(MMonCommand *m, return prepare_unset_flag(m, CEPH_OSDMAP_NOIN); else if (key == "nobackfill") return prepare_unset_flag(m, CEPH_OSDMAP_NOBACKFILL); + else if (key == "norebalance") + return prepare_unset_flag(m, CEPH_OSDMAP_NOREBALANCE); else if (key == "norecover") return prepare_unset_flag(m, CEPH_OSDMAP_NORECOVER); else if (key == "noscrub") diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index adc10a05631..d4c45110cd4 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -2359,6 +2359,8 @@ string OSDMap::get_flag_string(unsigned f) s += ",noin"; if (f & CEPH_OSDMAP_NOBACKFILL) s += ",nobackfill"; + if (f & CEPH_OSDMAP_NOREBALANCE) + s += ",norebalance"; if (f & CEPH_OSDMAP_NORECOVER) s += ",norecover"; if (f & CEPH_OSDMAP_NOSCRUB) diff --git a/src/osd/PG.cc b/src/osd/PG.cc index d7d2c7fe607..d1752e8fe08 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -6413,7 +6413,8 @@ boost::statechart::result PG::RecoveryState::Active::react(const ActMap&) } if (!pg->is_clean() && - !pg->get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL)) { + !pg->get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL) && + (!pg->get_osdmap()->test_flag(CEPH_OSDMAP_NOREBALANCE) || pg->is_degraded())) { pg->osd->queue_for_recovery(pg); } return forward_event(); diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 93f4f72b88d..c1102ece015 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -10661,6 +10661,10 @@ bool ReplicatedPG::start_recovery_ops( if (get_osdmap()->test_flag(CEPH_OSDMAP_NOBACKFILL)) { dout(10) << "deferring backfill due to NOBACKFILL" << dendl; deferred_backfill = true; + } else if (get_osdmap()->test_flag(CEPH_OSDMAP_NOREBALANCE) && + !is_degraded()) { + dout(10) << "deferring backfill due to NOREBALANCE" << dendl; + deferred_backfill = true; } else if (!backfill_reserved) { dout(10) << "deferring backfill due to !backfill_reserved" << dendl; if (!backfill_reserving) { From 484a147e9ee1a988bf752b9bb08d8ad4a897fa64 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 5 Feb 2015 12:34:42 +0800 Subject: [PATCH 2/2] osd: apply affinity if any affinity is not default --- src/osd/OSDMap.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index d4c45110cd4..e93040c33c0 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -1570,6 +1570,7 @@ void OSDMap::_apply_primary_affinity(ps_t seed, if (*p != CRUSH_ITEM_NONE && (*osd_primary_affinity)[*p] != CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) { any = true; + break; } } if (!any) @@ -2370,7 +2371,7 @@ string OSDMap::get_flag_string(unsigned f) if (f & CEPH_OSDMAP_NOTIERAGENT) s += ",notieragent"; if (s.length()) - s = s.erase(0, 1); + s.erase(0, 1); return s; }