RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-19 17:41:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Addressing CR comments from tchaikov (Kefu Chai).

Browse Source 
Signed-off-by: Yan Lei <yongyou.yl@alibaba-inc.com>
This commit is contained in:
ownedu 2017-10-01 17:07:53 +08:00
parent 92c3499f7b
commit 303e640c74
4 changed files with 19 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/msg/async/rdma/Infiniband.cc
View File

@ -166,9 +166,7 @@ Infiniband::QueuePair::QueuePair(
max_send_wr(tx_queue_len),
max_recv_wr(rx_queue_len),
q_key(q_key),
dead(false),
tx_wr(0),
tx_wc(0)
dead(false)
{
initial_psn = lrand48() & 0xffffff;
if (type != IBV_QPT_RC && type != IBV_QPT_UD && type != IBV_QPT_RAW_PACKET) {

10
src/msg/async/rdma/Infiniband.h
View File

@ -465,14 +465,8 @@ class Infiniband {
* Return true if the queue pair is in an error state, false otherwise.
*/
bool is_error() const;
/**
* Add Tx work request and completion counters.
*/
void add_tx_wr(uint32_t amt) { tx_wr += amt; }
void add_tx_wc(uint32_t amt) { tx_wc += amt; }
/**
* Get Tx work request and completion counter values.
*/
uint32_t get_tx_wr() const { return tx_wr; }
uint32_t get_tx_wc() const { return tx_wc; }
ibv_qp* get_qp() const { return qp; }
@ -497,8 +491,8 @@ class Infiniband {
uint32_t max_recv_wr;
uint32_t q_key;
bool dead;
std::atomic<uint32_t> tx_wr; // atomic counter for successful Tx WQEs
std::atomic<uint32_t> tx_wc; // atomic counter for successful Tx CQEs
std::atomic<uint32_t> tx_wr = {0}; // counter for successful Tx WQEs
std::atomic<uint32_t> tx_wc = {0}; // counter for successful Tx CQEs
};
public:

2
src/msg/async/rdma/RDMAConnectedSocketImpl.cc
View File

@ -577,7 +577,6 @@ int RDMAConnectedSocketImpl::post_work_request(std::vector<Chunk*> &tx_buffers)
worker->perf_logger->inc(l_msgr_rdma_tx_failed);
return -errno;
}
// Update the Tx WQE counter
qp->add_tx_wr(num);
worker->perf_logger->inc(l_msgr_rdma_tx_chunks, tx_buffers.size());
ldout(cct, 20) << __func__ << " qp state is " << Infiniband::qp_state_string(qp->get_state()) << dendl;
@ -599,7 +598,6 @@ void RDMAConnectedSocketImpl::fin() {
worker->perf_logger->inc(l_msgr_rdma_tx_failed);
return ;
}
// Update the Tx WQE counter
qp->add_tx_wr(1);
}

30
src/msg/async/rdma/RDMAStack.cc
View File

@ -244,13 +244,15 @@ void RDMADispatcher::polling()
perf_logger->set(l_msgr_rdma_inflight_tx_chunks, inflight);
if (num_dead_queue_pair) {
Mutex::Locker l(lock); // FIXME reuse dead qp because creating one qp costs 1 ms
for (auto idx = 0; idx < dead_queue_pairs.size(); idx++) {
for (auto &i : dead_queue_pairs) {
// Bypass QPs that do not collect all Tx completions yet.
if (dead_queue_pairs.at(idx)->get_tx_wc() != dead_queue_pairs.at(idx)->get_tx_wr())
if (i->get_tx_wc() != i->get_tx_wr())
continue;
ldout(cct, 10) << __func__ << " finally delete qp=" << dead_queue_pairs.at(idx) << dendl;
delete dead_queue_pairs.at(idx);
dead_queue_pairs.erase(dead_queue_pairs.begin() + idx);
ldout(cct, 10) << __func__ << " finally delete qp=" << i << dendl;
delete i;
auto it = std::find(dead_queue_pairs.begin(), dead_queue_pairs.end(), i);
if (it != dead_queue_pairs.end())
dead_queue_pairs.erase(it);
perf_logger->dec(l_msgr_rdma_active_queue_pair);
--num_dead_queue_pair;
}
@ -341,15 +343,15 @@ Infiniband::QueuePair* RDMADispatcher::get_qp(uint32_t qp)
Mutex::Locker l(lock);
// Try to find the QP in qp_conns firstly.
auto it = qp_conns.find(qp);
if (it == qp_conns.end()) {
// Try again in dead_queue_pairs.
for(auto dead_qp = dead_queue_pairs.begin(); dead_qp != dead_queue_pairs.end(); dead_qp++) {
if ((*dead_qp)->get_local_qp_number() == qp)
return *dead_qp;
}
return nullptr;
}
return it->second.first;
if (it != qp_conns.end())
return it->second.first;
// Try again in dead_queue_pairs.
for (auto &i: dead_queue_pairs)
if (i->get_local_qp_number() == qp)
return i;
return nullptr;
}
void RDMADispatcher::erase_qpn_lockless(uint32_t qpn)