From 96d6bf22a5dbf14da1de0ee6128585b9dd9d60d8 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 18 Nov 2021 15:46:06 -0500 Subject: [PATCH] osd/PeeringState: separate history's pruub from pg's (pruub = prior_readable_until_ub [upper bound]) During peering, a primary may conclude that it does not need to wait for the prior interval(s)' read lease because it will query all such osds. However, it is dangerous to reflect that local inference about future peering effects in the info.history, which is freely shared with other OSDs. For example, if the primary cleared the history pruub, shared it, and then failed, the next primary may conclude that it does not need to wait for the lease to expire. Instead, track the pruub in the conventional way. Only at the end of peering do we clear it (if there are no prior_interval_down_osds) before ending our final activate infos out, when we have already talked to the peer osds and they know the prior interval has finished. Fixes: https://tracker.ceph.com/issues/53326 Signed-off-by: Sage Weil --- src/osd/PeeringState.cc | 9 +++++++-- src/osd/PeeringState.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index b5e6b69573e..03688fd11aa 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -2721,6 +2721,11 @@ void PeeringState::activate( info.purged_snaps.swap(purged); // start up replicas + if (prior_readable_down_osds.empty()) { + dout(10) << __func__ << " no prior_readable_down_osds to wait on, clearing ub" + << dendl; + clear_prior_readable_until_ub(); + } info.history.refresh_prior_readable_until_ub(pl->get_mnow(), prior_readable_until_ub); @@ -6697,10 +6702,10 @@ PeeringState::GetInfo::GetInfo(my_context ctx) prior_set = ps->build_prior(); ps->prior_readable_down_osds = prior_set.down; + if (ps->prior_readable_down_osds.empty()) { - psdout(10) << " no prior_set down osds, clearing prior_readable_until_ub" + psdout(10) << " no prior_set down osds, will clear prior_readable_until_ub before activating" << dendl; - ps->clear_prior_readable_until_ub(); } ps->reset_min_peer_features(); diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h index 5e676613ca5..3cec0f2daa3 100644 --- a/src/osd/PeeringState.h +++ b/src/osd/PeeringState.h @@ -2096,6 +2096,7 @@ public: void clear_prior_readable_until_ub() { prior_readable_until_ub = ceph::signedspan::zero(); prior_readable_down_osds.clear(); + info.history.prior_readable_until_ub = ceph::signedspan::zero(); } void renew_lease(ceph::signedspan now) {