From 250677c965365edf3ecd24ef73700fc6d992ea42 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 Jun 2014 16:44:53 -0700 Subject: [PATCH 1/2] osd/OSDMap: make get_features() take an entity type Make the helper that returns what features are required of the OSDMap take an entity type argument, as the required features may vary between components in the cluster. Backport: firefly Signed-off-by: Sage Weil --- src/mon/OSDMonitor.cc | 8 ++++---- src/osd/OSD.cc | 37 +++++++++++++++++++++++++------------ src/osd/OSDMap.cc | 7 ++++--- src/osd/OSDMap.h | 3 ++- src/osd/PG.cc | 3 ++- 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 88c147c6760..ee6209a132e 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -283,15 +283,14 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap) void OSDMonitor::update_msgr_features() { - uint64_t mask; - uint64_t features = osdmap.get_features(&mask); - set types; types.insert((int)entity_name_t::TYPE_OSD); types.insert((int)entity_name_t::TYPE_CLIENT); types.insert((int)entity_name_t::TYPE_MDS); types.insert((int)entity_name_t::TYPE_MON); for (set::iterator q = types.begin(); q != types.end(); ++q) { + uint64_t mask; + uint64_t features = osdmap.get_features(*q, &mask); if ((mon->messenger->get_policy(*q).features_required & mask) != features) { dout(0) << "crush map has features " << features << ", adjusting msgr requires" << dendl; Messenger::Policy p = mon->messenger->get_policy(*q); @@ -1182,7 +1181,8 @@ bool OSDMonitor::preprocess_boot(MOSDBoot *m) assert(m->get_orig_source_inst().name.is_osd()); // check if osd has required features to boot - if ((osdmap.get_features(NULL) & CEPH_FEATURE_OSD_ERASURE_CODES) && + if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) & + CEPH_FEATURE_OSD_ERASURE_CODES) && !(m->get_connection()->get_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) { dout(0) << __func__ << " osdmap requires Erasure Codes but osd at " << m->get_orig_source_inst() diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index dd4004ca021..90ce6551ea9 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -5977,11 +5977,10 @@ void OSD::check_osdmap_features(ObjectStore *fs) // current memory location, and setting or clearing bits in integer // fields, and we are the only writer, this is not a problem. - uint64_t mask; - uint64_t features = osdmap->get_features(&mask); - { Messenger::Policy p = client_messenger->get_default_policy(); + uint64_t mask; + uint64_t features = osdmap->get_features(entity_name_t::TYPE_CLIENT, &mask); if ((p.features_required & mask) != features) { dout(0) << "crush map has features " << features << ", adjusting msgr requires for clients" << dendl; @@ -5989,25 +5988,39 @@ void OSD::check_osdmap_features(ObjectStore *fs) client_messenger->set_default_policy(p); } } + { + Messenger::Policy p = cluster_messenger->get_policy(entity_name_t::TYPE_MON); + uint64_t mask; + uint64_t features = osdmap->get_features(entity_name_t::TYPE_MON, &mask); + if ((p.features_required & mask) != features) { + dout(0) << "crush map has features " << features + << ", adjusting msgr requires for mons" << dendl; + p.features_required = (p.features_required & ~mask) | features; + client_messenger->set_policy(entity_name_t::TYPE_MON, p); + } + } { Messenger::Policy p = cluster_messenger->get_policy(entity_name_t::TYPE_OSD); + uint64_t mask; + uint64_t features = osdmap->get_features(entity_name_t::TYPE_OSD, &mask); + if ((p.features_required & mask) != features) { dout(0) << "crush map has features " << features << ", adjusting msgr requires for osds" << dendl; p.features_required = (p.features_required & ~mask) | features; cluster_messenger->set_policy(entity_name_t::TYPE_OSD, p); } - } - if ((features & CEPH_FEATURE_OSD_ERASURE_CODES) && + if ((features & CEPH_FEATURE_OSD_ERASURE_CODES) && !fs->get_allow_sharded_objects()) { - dout(0) << __func__ << " enabling on-disk ERASURE CODES compat feature" << dendl; - superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); - ObjectStore::Transaction *t = new ObjectStore::Transaction; - write_superblock(*t); - int err = store->queue_transaction_and_cleanup(NULL, t); - assert(err == 0); - fs->set_allow_sharded_objects(); + dout(0) << __func__ << " enabling on-disk ERASURE CODES compat feature" << dendl; + superblock.compat_features.incompat.insert(CEPH_OSD_FEATURE_INCOMPAT_SHARDS); + ObjectStore::Transaction *t = new ObjectStore::Transaction; + write_superblock(*t); + int err = store->queue_transaction_and_cleanup(NULL, t); + assert(err == 0); + fs->set_allow_sharded_objects(); + } } } diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 4c27f7ba7c3..cb83b88fafd 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -958,7 +958,7 @@ bool OSDMap::find_osd_on_ip(const entity_addr_t& ip) const } -uint64_t OSDMap::get_features(uint64_t *pmask) const +uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const { uint64_t features = 0; // things we actually have uint64_t mask = 0; // things we could have @@ -986,8 +986,9 @@ uint64_t OSDMap::get_features(uint64_t *pmask) const features |= CEPH_FEATURE_OSD_CACHEPOOL; } } - mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL | - CEPH_FEATURE_OSD_ERASURE_CODES; + mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL; + if (entity_type != CEPH_ENTITY_TYPE_CLIENT) + mask |= CEPH_FEATURE_OSD_ERASURE_CODES; if (osd_primary_affinity) { for (int i = 0; i < max_osd; ++i) { diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index e5ffc0e1c73..fcb3e26d847 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -534,10 +534,11 @@ public: /** * get feature bits required by the current structure * + * @param entity_type [in] what entity type we are asking about * @param mask [out] set of all possible map-related features we could set * @return feature bits used by this map */ - uint64_t get_features(uint64_t *mask) const; + uint64_t get_features(int entity_type, uint64_t *mask) const; /** * get intersection of features supported by up osds diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 42f9aea26fe..4c2c786914f 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -397,7 +397,8 @@ bool PG::search_for_missing( if (found_missing && num_unfound_before != missing_loc.num_unfound()) publish_stats_to_osd(); if (found_missing && - (get_osdmap()->get_features(NULL) & CEPH_FEATURE_OSD_ERASURE_CODES)) { + (get_osdmap()->get_features(CEPH_ENTITY_TYPE_OSD, NULL) & + CEPH_FEATURE_OSD_ERASURE_CODES)) { pg_info_t tinfo(oinfo); tinfo.pgid.shard = pg_whoami.shard; (*(ctx->info_map))[from.osd].push_back( From 3fe1699f9620280d0070cfe6f01cfeb2332e7470 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 6 Jun 2014 13:31:29 -0700 Subject: [PATCH 2/2] osd/OSDMap: do not require ERASURE_CODE feature of clients Just because an EC pool exists in the cluster does not mean tha tthe client has to support the feature: 1) The way client IO is initiated is no different for EC pools than for replicated pools. 2) People may add an EC pool to an existing cluster with old clients and locking those old clients out is very rude when they are not using the new pool. 3) The only direct client user of EC pools right now is rgw, and the new versions already need to support various other features like CRUSH_V2 in order to work. These features are present in new kernels. Fixes: #8556 Backport: firefly Signed-off-by: Sage Weil --- src/osd/OSDMap.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index cb83b88fafd..8d2a346fb64 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -978,7 +978,8 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const if (p->second.flags & pg_pool_t::FLAG_HASHPSPOOL) { features |= CEPH_FEATURE_OSDHASHPSPOOL; } - if (p->second.is_erasure()) { + if (p->second.is_erasure() && + entity_type != CEPH_ENTITY_TYPE_CLIENT) { // not for clients features |= CEPH_FEATURE_OSD_ERASURE_CODES; } if (!p->second.tiers.empty() ||