diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index c8078787df2..e83e29b8bb6 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -36,6 +36,7 @@ #include "include/types.h" #include "include/compat.h" +#include "include/random.h" #include "OSD.h" #include "OSDMap.h" @@ -169,8 +170,6 @@ #define dout_prefix _prefix(_dout, whoami, get_osdmap_epoch()) -const double OSD::OSD_TICK_INTERVAL = 1.0; - static ostream& _prefix(std::ostream* _dout, int whoami, epoch_t epoch) { return *_dout << "osd." << whoami << " " << epoch << " "; } @@ -603,8 +602,8 @@ void OSDService::promote_throttle_recalibrate() promote_probability_millis = prob; // set hard limits for this interval to mitigate stampedes - promote_max_objects = target_obj_sec * OSD::OSD_TICK_INTERVAL * 2; - promote_max_bytes = target_bytes_sec * OSD::OSD_TICK_INTERVAL * 2; + promote_max_objects = target_obj_sec * osd->OSD_TICK_INTERVAL * 2; + promote_max_bytes = target_bytes_sec * osd->OSD_TICK_INTERVAL * 2; } // ------------------------------------- @@ -1978,6 +1977,14 @@ OSD::~OSD() delete store; } +double OSD::get_tick_interval() const +{ + // vary +/- 5% to avoid scrub scheduling livelocks + constexpr auto delta = 0.05; + return (OSD_TICK_INTERVAL * + ceph::util::generate_random_number(1.0 - delta, 1.0 + delta)); +} + void cls_initialize(ClassHandler *ch); void OSD::handle_signal(int signum) @@ -2606,10 +2613,12 @@ int OSD::init() heartbeat_thread.create("osd_srv_heartbt"); // tick - tick_timer.add_event_after(cct->_conf->osd_heartbeat_interval, new C_Tick(this)); + tick_timer.add_event_after(get_tick_interval(), + new C_Tick(this)); { Mutex::Locker l(tick_timer_lock); - tick_timer_without_osd_lock.add_event_after(cct->_conf->osd_heartbeat_interval, new C_Tick_WithoutOSDLock(this)); + tick_timer_without_osd_lock.add_event_after(get_tick_interval(), + new C_Tick_WithoutOSDLock(this)); } osd_lock.Unlock(); @@ -4813,7 +4822,7 @@ void OSD::tick() do_waiters(); - tick_timer.add_event_after(OSD_TICK_INTERVAL, new C_Tick(this)); + tick_timer.add_event_after(get_tick_interval(), new C_Tick(this)); } void OSD::tick_without_osd_lock() @@ -4884,7 +4893,8 @@ void OSD::tick_without_osd_lock() mgrc.update_daemon_health(get_health_metrics()); service.kick_recovery_queue(); - tick_timer_without_osd_lock.add_event_after(OSD_TICK_INTERVAL, new C_Tick_WithoutOSDLock(this)); + tick_timer_without_osd_lock.add_event_after(get_tick_interval(), + new C_Tick_WithoutOSDLock(this)); } // Usage: diff --git a/src/osd/OSD.h b/src/osd/OSD.h index e435b99000a..18739f52dfa 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1237,7 +1237,8 @@ public: protected: - static const double OSD_TICK_INTERVAL; // tick interval for tick_timer and tick_timer_without_osd_lock + const double OSD_TICK_INTERVAL = { 1.0 }; + double get_tick_interval() const; AuthAuthorizeHandlerRegistry *authorize_handler_cluster_registry; AuthAuthorizeHandlerRegistry *authorize_handler_service_registry;