From 48f60e7f274de9d76499816a528eff859bb161e3 Mon Sep 17 00:00:00 2001 From: "Yan, Zheng" Date: Wed, 2 May 2018 10:23:33 +0800 Subject: [PATCH] mds: properly reconnect client caps after loading inodes Commit e43c02d6 "mds: filter out blacklisted clients when importing caps" makes MDCache::process_imported_caps() ignore clients that are not in MDCache::rejoin_imported_session_map. The map does not contain clients from which mds has received reconnect messages. This causes some client caps (corresponding inodes were not in cache when mds was in reconnect state) to get dropped. Signed-off-by: "Yan, Zheng" --- src/mds/MDCache.cc | 34 +++++++++++++++++----------------- src/mds/MDCache.h | 9 ++++++--- src/mds/Server.cc | 1 + 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index eb07de00ed1..fcd8b5ac573 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -3239,7 +3239,7 @@ void MDCache::handle_resolve(MMDSResolve *m) Session *session = mds->sessionmap.get_session(entity_name_t::CLIENT(q->first.v)); if (session) - rejoin_imported_client_map.emplace(q->first, session->info.inst); + rejoin_client_map.emplace(q->first, session->info.inst); } // will process these caps in rejoin stage @@ -4420,7 +4420,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak) assert(gather_locks.empty()); // check cap exports. - rejoin_imported_client_map.insert(weak->client_map.begin(), weak->client_map.end()); + rejoin_client_map.insert(weak->client_map.begin(), weak->client_map.end()); for (auto p = weak->cap_exports.begin(); p != weak->cap_exports.end(); ++p) { CInode *in = get_inode(p->first); @@ -5376,19 +5376,19 @@ void MDCache::rejoin_open_ino_finish(inodeno_t ino, int ret) class C_MDC_RejoinSessionsOpened : public MDCacheLogContext { public: - map > imported_session_map; + map > session_map; C_MDC_RejoinSessionsOpened(MDCache *c) : MDCacheLogContext(c) {} void finish(int r) override { assert(r == 0); - mdcache->rejoin_open_sessions_finish(imported_session_map); + mdcache->rejoin_open_sessions_finish(session_map); } }; -void MDCache::rejoin_open_sessions_finish(map >& imported_session_map) +void MDCache::rejoin_open_sessions_finish(map >& session_map) { dout(10) << "rejoin_open_sessions_finish" << dendl; - mds->server->finish_force_open_sessions(imported_session_map); - rejoin_imported_session_map.swap(imported_session_map); + mds->server->finish_force_open_sessions(session_map); + rejoin_session_map.swap(session_map); if (rejoin_gather.empty()) rejoin_gather_finish(); } @@ -5450,14 +5450,14 @@ bool MDCache::process_imported_caps() // called by rejoin_gather_finish() ? if (rejoin_gather.count(mds->get_nodeid()) == 0) { - if (!rejoin_imported_client_map.empty() && - rejoin_imported_session_map.empty()) { + if (!rejoin_client_map.empty() && + rejoin_session_map.empty()) { C_MDC_RejoinSessionsOpened *finish = new C_MDC_RejoinSessionsOpened(this); - version_t pv = mds->server->prepare_force_open_sessions(rejoin_imported_client_map, - finish->imported_session_map); - mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_imported_client_map), finish); + version_t pv = mds->server->prepare_force_open_sessions(rejoin_client_map, + finish->session_map); + mds->mdlog->start_submit_entry(new ESessions(pv, rejoin_client_map), finish); mds->mdlog->flush(); - rejoin_imported_client_map.clear(); + rejoin_client_map.clear(); return true; } @@ -5470,8 +5470,8 @@ bool MDCache::process_imported_caps() for (map::iterator q = p->second.second.begin(); q != p->second.second.end(); ++q) { - auto r = rejoin_imported_session_map.find(q->first); - if (r == rejoin_imported_session_map.end()) + auto r = rejoin_session_map.find(q->first); + if (r == rejoin_session_map.end()) continue; Session *session = r->second.first; @@ -5509,8 +5509,8 @@ bool MDCache::process_imported_caps() for (auto q = p->second.begin(); q != p->second.end(); ++q) { Session *session; { - auto r = rejoin_imported_session_map.find(q->first); - session = (r != rejoin_imported_session_map.end() ? r->second.first : nullptr); + auto r = rejoin_session_map.find(q->first); + session = (r != rejoin_session_map.end() ? r->second.first : nullptr); } for (auto r = q->second.begin(); r != q->second.end(); ++r) { diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h index 99f0b13a835..3113ac8303c 100644 --- a/src/mds/MDCache.h +++ b/src/mds/MDCache.h @@ -552,8 +552,8 @@ protected: set rejoin_ack_gather; // nodes from whom i need a rejoin ack map > > rejoin_imported_caps; map > > rejoin_slave_exports; - map rejoin_imported_client_map; - map > rejoin_imported_session_map; + map rejoin_client_map; + map > rejoin_session_map; map > > cap_exports; // ino -> target, client -> capex @@ -603,6 +603,9 @@ public: mds_rank_t frommds=MDS_RANK_NONE) { cap_imports[ino][client][frommds] = icr; } + void rejoin_recovered_client(client_t client, const entity_inst_t& inst) { + rejoin_client_map.emplace(client, inst); + } bool rejoin_has_cap_reconnect(inodeno_t ino) const { return cap_imports.count(ino); } @@ -657,7 +660,7 @@ public: friend class C_MDC_RejoinSessionsOpened; void rejoin_open_ino_finish(inodeno_t ino, int ret); void rejoin_prefetch_ino_finish(inodeno_t ino, int ret); - void rejoin_open_sessions_finish(map >& imported_session_map); + void rejoin_open_sessions_finish(map >& session_map); bool process_imported_caps(); void choose_lock_states_and_reconnect_caps(); void prepare_realm_split(SnapRealm *realm, client_t client, inodeno_t ino, diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 347a5c33ff4..d93ad9d125b 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -1013,6 +1013,7 @@ void Server::handle_client_reconnect(MClientReconnect *m) mdcache->rejoin_recovered_caps(p->first, from, p->second, MDS_RANK_NONE); } } + mdcache->rejoin_recovered_client(session->get_client(), session->info.inst); // remove from gather set client_reconnect_gather.erase(from);