From cb9262abd7fd5f0a9f583bd34e4c425a049e56ce Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Mon, 20 Oct 2014 13:29:13 -0700 Subject: [PATCH] Objecter: resend linger ops on any interval change Watch/notify ops need to be resent after a pg split occurs, as well as a few other circumstances that the existing objecter checks did not catch. Refactor the check the OSD uses for this to add a version taking the more basic types instead of the whole OSD map, and stash the needed info when an op is sent. Fixes: #9806 Backport: giant, firefly, dumpling Signed-off-by: Josh Durgin --- src/osd/osd_types.cc | 41 ++++++++++++++++++++++++++++++++++------- src/osd/osd_types.h | 19 +++++++++++++++++++ src/osdc/Objecter.cc | 39 ++++++++++++++++++++++++++++++++------- src/osdc/Objecter.h | 13 ++++++++++--- 4 files changed, 95 insertions(+), 17 deletions(-) diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 47cb37bc084..ed06c4fd6e1 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -2329,18 +2329,45 @@ bool pg_interval_t::is_new_interval( int new_up_primary, const vector &old_up, const vector &new_up, - OSDMapRef osdmap, - OSDMapRef lastmap, + int old_min_size, + int new_min_size, + unsigned old_pg_num, + unsigned new_pg_num, pg_t pgid) { return old_acting_primary != new_acting_primary || new_acting != old_acting || old_up_primary != new_up_primary || new_up != old_up || - (!(lastmap->get_pools().count(pgid.pool()))) || - (lastmap->get_pools().find(pgid.pool())->second.min_size != - osdmap->get_pools().find(pgid.pool())->second.min_size) || - pgid.is_split(lastmap->get_pg_num(pgid.pool()), - osdmap->get_pg_num(pgid.pool()), 0); + old_min_size != new_min_size || + pgid.is_split(old_pg_num, new_pg_num, 0); +} + +bool pg_interval_t::is_new_interval( + int old_acting_primary, + int new_acting_primary, + const vector &old_acting, + const vector &new_acting, + int old_up_primary, + int new_up_primary, + const vector &old_up, + const vector &new_up, + OSDMapRef osdmap, + OSDMapRef lastmap, + pg_t pgid) { + return !(lastmap->get_pools().count(pgid.pool())) || + is_new_interval(old_acting_primary, + new_acting_primary, + old_acting, + new_acting, + old_up_primary, + new_up_primary, + old_up, + new_up, + lastmap->get_pools().find(pgid.pool())->second.min_size, + osdmap->get_pools().find(pgid.pool())->second.min_size, + lastmap->get_pg_num(pgid.pool()), + osdmap->get_pg_num(pgid.pool()), + pgid); } bool pg_interval_t::check_new_interval( diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 9840bf16cfe..988cebe6959 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -1773,6 +1773,25 @@ struct pg_interval_t { void dump(Formatter *f) const; static void generate_test_instances(list& o); + /** + * Determines whether there is an interval change + */ + static bool is_new_interval( + int old_acting_primary, + int new_acting_primary, + const vector &old_acting, + const vector &new_acting, + int old_up_primary, + int new_up_primary, + const vector &old_up, + const vector &new_up, + int old_min_size, + int new_min_size, + unsigned old_pg_num, + unsigned new_pg_num, + pg_t pgid + ); + /** * Determines whether there is an interval change */ diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc index 035127acc99..048ecc7f9df 100644 --- a/src/osdc/Objecter.cc +++ b/src/osdc/Objecter.cc @@ -2050,9 +2050,29 @@ int Objecter::_calc_target(op_target_t *t, bool any_change) return RECALC_OP_TARGET_POOL_DNE; } } - int primary; - vector acting; - osdmap->pg_to_acting_osds(pgid, &acting, &primary); + + int min_size = pi->min_size; + unsigned pg_num = pi->get_pg_num(); + int up_primary, acting_primary; + vector up, acting; + osdmap->pg_to_up_acting_osds(pgid, &up, &up_primary, + &acting, &acting_primary); + if (any_change && pg_interval_t::is_new_interval( + t->acting_primary, + acting_primary, + t->acting, + acting, + t->up_primary, + up_primary, + t->up, + up, + t->min_size, + min_size, + t->pg_num, + pg_num, + pi->raw_pg_to_pg(pgid))) { + force_resend = true; + } bool need_resend = false; @@ -2064,15 +2084,20 @@ int Objecter::_calc_target(op_target_t *t, bool any_change) if (t->pgid != pgid || is_pg_changed( - t->primary, t->acting, primary, acting, t->used_replica || any_change) || + t->acting_primary, t->acting, acting_primary, acting, + t->used_replica || any_change) || force_resend) { t->pgid = pgid; t->acting = acting; - t->primary = primary; + t->acting_primary = acting_primary; + t->up_primary = up_primary; + t->up = up; + t->min_size = min_size; + t->pg_num = pg_num; ldout(cct, 10) << __func__ << " " << " pgid " << pgid << " acting " << acting << dendl; t->used_replica = false; - if (primary == -1) { + if (acting_primary == -1) { t->osd = -1; } else { int osd; @@ -2108,7 +2133,7 @@ int Objecter::_calc_target(op_target_t *t, bool any_change) assert(best >= 0); osd = acting[best]; } else { - osd = primary; + osd = acting_primary; } t->osd = osd; } diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h index b45f40615b2..8bf1350cd3f 100644 --- a/src/osdc/Objecter.h +++ b/src/osdc/Objecter.h @@ -1078,8 +1078,12 @@ public: pg_t base_pgid; ///< explciti pg target, if any pg_t pgid; ///< last pg we mapped to - vector acting; ///< acting for last pg we mapped to - int primary; ///< primary for last pg we mapped to + unsigned pg_num; ///< last pg_num we mapped to + vector up; ///< set of up osds for last pg we mapped to + vector acting; ///< set of acting osds for last pg we mapped to + int up_primary; ///< primary for last pg we mapped to based on the up set + int acting_primary; ///< primary for last pg we mapped to based on the acting set + int min_size; ///< the min size of the pool when were were last mapped bool used_replica; bool paused; @@ -1091,7 +1095,10 @@ public: base_oid(oid), base_oloc(oloc), precalc_pgid(false), - primary(-1), + pg_num(0), + up_primary(-1), + acting_primary(-1), + min_size(-1), used_replica(false), paused(false), osd(-1)