From 74fc3f6b83812acdd2733c61517a3efc68e0c57d Mon Sep 17 00:00:00 2001
From: Adir Lev <adirl@mellanox.com>
Date: Thu, 9 Feb 2017 15:52:22 +0000
Subject: [PATCH] msg/async/rdma: check if exp verbs avail

issue: 975125

Change-Id: I9daa168ca0299887a7238a688508c773b98abde9
Signed-off-by: Adir Lev <adirl@mellanox.com>
Signed-off-by: Oren Duer <oren@mellanox.com>
---
 cmake/modules/Findrdma.cmake     | 11 +++++++++++
 src/include/config-h.in.cmake    |  3 +++
 src/msg/async/rdma/Infiniband.cc | 16 ++++++++++++++++
 3 files changed, 30 insertions(+)

diff --git a/cmake/modules/Findrdma.cmake b/cmake/modules/Findrdma.cmake
index 2799247ba1f..eb31f7922c9 100644
--- a/cmake/modules/Findrdma.cmake
+++ b/cmake/modules/Findrdma.cmake
@@ -20,6 +20,17 @@ endif ()
 
 if (RDMA_FOUND)
   message(STATUS "Found libibverbs: ${RDMA_LIBRARY}")
+
+  include(CheckCXXSourceCompiles)
+  CHECK_CXX_SOURCE_COMPILES("
+    #include <infiniband/verbs.h>
+    int main() {
+      struct ibv_context* ctxt;
+      struct ibv_exp_gid_attr gid_attr;
+      ibv_exp_query_gid_attr(ctxt, 1, 0, &gid_attr);
+      return 0;
+    } " HAVE_IBV_EXP)
+
 else ()
   message(STATUS "Not Found libibverbs: ${RDMA_LIBRARY}")
   if (RDMA_FIND_REQUIRED)
diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake
index d731a9b6c1f..575daed610b 100644
--- a/src/include/config-h.in.cmake
+++ b/src/include/config-h.in.cmake
@@ -130,6 +130,9 @@
 /* AsyncMessenger RDMA conditional compilation */
 #cmakedefine HAVE_RDMA
 
+/* ibverbs experimental conditional compilation */
+#cmakedefine HAVE_IBV_EXP
+
 /* define if embedded enabled */
 #cmakedefine WITH_EMBEDDED
 
diff --git a/src/msg/async/rdma/Infiniband.cc b/src/msg/async/rdma/Infiniband.cc
index e6d9cd297da..40a69bcacfa 100644
--- a/src/msg/async/rdma/Infiniband.cc
+++ b/src/msg/async/rdma/Infiniband.cc
@@ -29,10 +29,12 @@ static const uint32_t CQ_DEPTH = 30000;
 
 Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt), port_num(ipn), port_attr(new ibv_port_attr)
 {
+#ifdef HAVE_IBV_EXP
   union ibv_gid cgid;
   struct ibv_exp_gid_attr gid_attr;
   bool malformed = false;
 
+  ldout(cct,1) << __func__ << " using experimental verbs for gid" << dendl;
   int r = ibv_query_port(ctxt, port_num, port_attr);
   if (r == -1) {
     lderr(cct) << __func__  << " query port failed  " << cpp_strerror(errno) << dendl;
@@ -87,6 +89,20 @@ Port::Port(CephContext *cct, struct ibv_context* ictxt, uint8_t ipn): ctxt(ictxt
     lderr(cct) << __func__ << " Requested local GID was not found in GID table" << dendl;
     ceph_abort();
   }
+#else
+  int r = ibv_query_port(ctxt, port_num, port_attr);
+  if (r == -1) {
+    lderr(cct) << __func__  << " query port failed  " << cpp_strerror(errno) << dendl;
+    ceph_abort();
+  }
+
+  lid = port_attr->lid;
+  r = ibv_query_gid(ctxt, port_num, 0, &gid);
+  if (r) {
+    lderr(cct) << __func__  << " query gid failed  " << cpp_strerror(errno) << dendl;
+    ceph_abort();
+  }
+#endif
 }