From 1cad0401cfc74482db4118199d9bca5e3c7b254d Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Tue, 1 Oct 2024 00:54:36 +0530 Subject: [PATCH 1/3] mon: add nvmeof healthchecks Add NVMeofGwMap::get_health_checks which raises NVMEOF_SINGLE_GATEWAY if any of the groups have 1 gateway. In NVMeofGwMon, call `encode_health` and `load_health` to register healthchecks. This will add nvmeof healthchecks to "ceph health" output. Signed-off-by: Vallari Agrawal --- doc/rados/operations/health-checks.rst | 12 ++++++++++++ src/mon/NVMeofGwMap.cc | 25 +++++++++++++++++++++++++ src/mon/NVMeofGwMap.h | 5 +++++ src/mon/NVMeofGwMon.cc | 6 ++++++ 4 files changed, 48 insertions(+) diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst index 81dafdf03e9..bbce958bebe 100644 --- a/doc/rados/operations/health-checks.rst +++ b/doc/rados/operations/health-checks.rst @@ -1633,6 +1633,18 @@ We encourage you to fix this by making the weights even on both dividing buckets This can be done by making sure the combined weight of the OSDs on each dividing bucket are the same. +NVMeoF Gateway +-------------- + +NVMOEF_SINGLE_GATEWAY +__________________________________ + +One of the gateway group has only one gateway. This is not ideal because it makes +high availability (HA) impossible with a single gatway in a group. This can lead to +problems with failover and failback operations for the NVMeoF gateway. + +It's recommended to have multiple NVMeoF gateways in a group. + Miscellaneous ------------- diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc index c01ea9e7103..c350622b7db 100755 --- a/src/mon/NVMeofGwMap.cc +++ b/src/mon/NVMeofGwMap.cc @@ -16,7 +16,9 @@ #include "NVMeofGwMon.h" #include "NVMeofGwMap.h" #include "OSDMonitor.h" +#include "mon/health_check.h" +using std::list; using std::map; using std::make_pair; using std::ostream; @@ -879,6 +881,29 @@ struct CMonRequestProposal : public Context { } }; +void NVMeofGwMap::get_health_checks(health_check_map_t *checks) const +{ + list detail; + for (const auto& created_map_pair: created_gws) { + const auto& group_key = created_map_pair.first; + auto& group = group_key.second; + const NvmeGwMonStates& gw_created_map = created_map_pair.second; + if ( gw_created_map.size() == 1) { + ostringstream ss; + ss << "NVMeoF Gateway Group '" << group << "' has 1 gateway." ; + detail.push_back(ss.str()); + } + } + if (!detail.empty()) { + ostringstream ss; + ss << detail.size() << " group(s) have only 1 nvmeof gateway" + << "; HA is not possible with single gateway."; + auto& d = checks->add("NVMEOF_SINGLE_GATEWAY", HEALTH_WARN, + ss.str(), detail.size()); + d.detail.swap(detail); + } +} + int NVMeofGwMap::blocklist_gw( const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId grpid, epoch_t &epoch, bool failover) diff --git a/src/mon/NVMeofGwMap.h b/src/mon/NVMeofGwMap.h index 267d85b10f9..5f657733012 100755 --- a/src/mon/NVMeofGwMap.h +++ b/src/mon/NVMeofGwMap.h @@ -27,6 +27,9 @@ #include "NVMeofGwTypes.h" using ceph::coarse_mono_clock; + +class health_check_map_t; + class Monitor; /*-------------------*/ class NVMeofGwMap @@ -140,6 +143,8 @@ public: decode(fsm_timers, bl); DECODE_FINISH(bl); } + + void get_health_checks(health_check_map_t *checks) const; }; #include "NVMeofGwSerialize.h" diff --git a/src/mon/NVMeofGwMon.cc b/src/mon/NVMeofGwMon.cc index d9e936e27df..25c14381964 100644 --- a/src/mon/NVMeofGwMon.cc +++ b/src/mon/NVMeofGwMon.cc @@ -176,6 +176,11 @@ void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t) << HAVE_FEATURE(mon.get_quorum_con_features(), NVMEOFHA) << dendl; put_version(t, pending_map.epoch, bl); put_last_committed(t, pending_map.epoch); + + //health + health_check_map_t checks; + pending_map.get_health_checks(&checks); + encode_health(checks, t); } void NVMeofGwMon::update_from_paxos(bool *need_bootstrap) @@ -188,6 +193,7 @@ void NVMeofGwMon::update_from_paxos(bool *need_bootstrap) bufferlist bl; int err = get_version(version, bl); ceph_assert(err == 0); + load_health(); auto p = bl.cbegin(); map.decode(p); From 0006599c9e1a6941e0a347f2c86e795f1fda56c7 Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Tue, 1 Oct 2024 11:40:23 +0530 Subject: [PATCH 2/3] mon: add warning NVMEOF_GATEWAY_DOWN In src/mon/NVMeofGwMap.cc, add warning NVMEOF_GATEWAY_DOWN when any gateway is in GW_UNAVAILABLE state. Signed-off-by: Vallari Agrawal --- doc/rados/operations/health-checks.rst | 11 ++++++++-- src/mon/NVMeofGwMap.cc | 30 ++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst index bbce958bebe..4ee02c04e1e 100644 --- a/doc/rados/operations/health-checks.rst +++ b/doc/rados/operations/health-checks.rst @@ -1636,8 +1636,8 @@ bucket are the same. NVMeoF Gateway -------------- -NVMOEF_SINGLE_GATEWAY -__________________________________ +NVMEOF_SINGLE_GATEWAY +_____________________ One of the gateway group has only one gateway. This is not ideal because it makes high availability (HA) impossible with a single gatway in a group. This can lead to @@ -1645,6 +1645,13 @@ problems with failover and failback operations for the NVMeoF gateway. It's recommended to have multiple NVMeoF gateways in a group. +NVMEOF_GATEWAY_DOWN +___________________ + +Some of the gateways are in the GW_UNAVAILABLE state. If a NVMeoF daemon has crashed, +the daemon log file (found at ``/var/log/ceph/``) may contain troubleshooting information. + + Miscellaneous ------------- diff --git a/src/mon/NVMeofGwMap.cc b/src/mon/NVMeofGwMap.cc index c350622b7db..b29606db998 100755 --- a/src/mon/NVMeofGwMap.cc +++ b/src/mon/NVMeofGwMap.cc @@ -883,7 +883,8 @@ struct CMonRequestProposal : public Context { void NVMeofGwMap::get_health_checks(health_check_map_t *checks) const { - list detail; + list singleGatewayDetail; + list gatewayDownDetail; for (const auto& created_map_pair: created_gws) { const auto& group_key = created_map_pair.first; auto& group = group_key.second; @@ -891,16 +892,33 @@ void NVMeofGwMap::get_health_checks(health_check_map_t *checks) const if ( gw_created_map.size() == 1) { ostringstream ss; ss << "NVMeoF Gateway Group '" << group << "' has 1 gateway." ; - detail.push_back(ss.str()); + singleGatewayDetail.push_back(ss.str()); + } + for (const auto& gw_created_pair: gw_created_map) { + const auto& gw_id = gw_created_pair.first; + const auto& gw_created = gw_created_pair.second; + if (gw_created.availability == gw_availability_t::GW_UNAVAILABLE) { + ostringstream ss; + ss << "NVMeoF Gateway '" << gw_id << "' is unavailable." ; + gatewayDownDetail.push_back(ss.str()); + } } } - if (!detail.empty()) { + if (!singleGatewayDetail.empty()) { ostringstream ss; - ss << detail.size() << " group(s) have only 1 nvmeof gateway" + ss << singleGatewayDetail.size() << " group(s) have only 1 nvmeof gateway" << "; HA is not possible with single gateway."; auto& d = checks->add("NVMEOF_SINGLE_GATEWAY", HEALTH_WARN, - ss.str(), detail.size()); - d.detail.swap(detail); + ss.str(), singleGatewayDetail.size()); + d.detail.swap(singleGatewayDetail); + } + if (!gatewayDownDetail.empty()) { + ostringstream ss; + ss << gatewayDownDetail.size() << " gateway(s) are in unavailable state" + << "; gateway might be down, try to redeploy."; + auto& d = checks->add("NVMEOF_GATEWAY_DOWN", HEALTH_WARN, + ss.str(), gatewayDownDetail.size()); + d.detail.swap(gatewayDownDetail); } } From 73d5c01ae628c36d4055b817954e04e8033d39af Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Tue, 15 Oct 2024 18:10:20 +0530 Subject: [PATCH 3/3] qa/suites/nvmeof: add nvmeof warnings to log-ignorelist Add NVMEOF_SINGLE_GATEWAY and NVMEOF_GATEWAY_DOWN warnings to nvmeof:thrash job's log-ignorelist Signed-off-by: Vallari Agrawal --- qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml | 2 ++ qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml | 3 +++ qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml | 3 +++ 3 files changed, 8 insertions(+) diff --git a/qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml b/qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml index e8f390c3b78..7f20f9f04a8 100644 --- a/qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml +++ b/qa/suites/nvmeof/basic/clusters/4-gateways-2-initiator.yaml @@ -28,3 +28,5 @@ overrides: mon: # cephadm can take up to 5 minutes to bring up remaining mons mon down mkfs grace: 300 + log-ignorelist: + - NVMEOF_SINGLE_GATEWAY diff --git a/qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml b/qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml index 6a5bd1d754e..19fa2ec605d 100644 --- a/qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml +++ b/qa/suites/nvmeof/thrash/thrashers/nvmeof_mon_thrash.yaml @@ -8,6 +8,9 @@ overrides: - out of quorum # nvmeof daemon thrashing - CEPHADM_FAILED_DAEMON + - NVMEOF_SINGLE_GATEWAY + - NVMEOF_GATEWAY_DOWN + - are in unavailable state - is in error state - failed cephadm daemon diff --git a/qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml b/qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml index 422c821536a..80bf0527715 100644 --- a/qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml +++ b/qa/suites/nvmeof/thrash/thrashers/nvmeof_thrash.yaml @@ -3,6 +3,9 @@ overrides: log-ignorelist: # nvmeof daemon thrashing - CEPHADM_FAILED_DAEMON + - NVMEOF_SINGLE_GATEWAY + - NVMEOF_GATEWAY_DOWN + - are in unavailable state - is in error state - failed cephadm daemon