diff --git a/CMakeLists.txt b/CMakeLists.txt index 3692dd30ffc..fb1885de519 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,11 +172,17 @@ if(WITH_STATIC_LIBSTDCXX) endif() endif() -option(WITH_RDMA "Enable RDMA in async messenger" ON) -if(WITH_RDMA) - find_package(rdma REQUIRED) - set(HAVE_RDMA ${RDMA_FOUND}) -endif(WITH_RDMA) +option(WITH_VERBS "Enable RDMA verbs in async messenger" ON) +if(WITH_VERBS) + find_package(verbs REQUIRED) + set(HAVE_VERBS ${VERBS_FOUND}) +endif(WITH_VERBS) + +option(WITH_RDMACM "Enable RDMA cm in async messenger" ON) +if(WITH_RDMACM) + find_package(rdmacm REQUIRED) + set(HAVE_RDMACM ${RDMACM_FOUND}) +endif(WITH_RDMACM) find_package(Backtrace) diff --git a/cmake/modules/Findrdma.cmake b/cmake/modules/Findrdma.cmake deleted file mode 100644 index dcc6cac825b..00000000000 --- a/cmake/modules/Findrdma.cmake +++ /dev/null @@ -1,53 +0,0 @@ -# - Find rdma -# Find the rdma library and includes -# -# RDMA_INCLUDE_DIR - where to find ibverbs.h, etc. -# RDMA_LIBRARIES - List of libraries when using ibverbs. -# RDMA_FOUND - True if ibverbs found. - -find_path(RDMA_INCLUDE_DIR infiniband/verbs.h) - -set(RDMA_NAMES ${RDMA_NAMES} ibverbs) -find_library(RDMA_LIBRARY NAMES ${RDMA_NAMES}) - -set(RDMACM_NAMES ${RDMACM_NAMES} rdmacm) -find_library(RDMACM_LIBRARY NAMES ${RDMACM_NAMES}) - -if (RDMA_INCLUDE_DIR AND RDMA_LIBRARY) - set(RDMA_FOUND TRUE) - set(RDMA_LIBRARIES ${RDMA_LIBRARY} ${RDMACM_LIBRARY}) -else () - set(RDMA_FOUND FALSE) - set( RDMA_LIBRARIES ) -endif () - -if (RDMA_FOUND) - message(STATUS "Found libibverbs: ${RDMA_LIBRARY}") - message(STATUS "Found librdmacm: ${RDMACM_LIBRARY}") - - include(CheckCXXSourceCompiles) - CHECK_CXX_SOURCE_COMPILES(" - #include - int main() { - struct ibv_context* ctxt; - struct ibv_exp_gid_attr gid_attr; - ibv_exp_query_gid_attr(ctxt, 1, 0, &gid_attr); - return 0; - } " HAVE_IBV_EXP) - -else () - message(STATUS "Not Found libibverbs: ${RDMA_LIBRARY}") - if (RDMA_FIND_REQUIRED) - message(STATUS "Looked for libibverbs named ${RDMA_NAMES}.") - message(FATAL_ERROR "Could NOT find libibverbs") - endif () -endif () - -# handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE if -# all listed variables are TRUE -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ibverbs DEFAULT_MSG RDMA_LIBRARIES RDMA_INCLUDE_DIR) - -mark_as_advanced( - RDMA_LIBRARY -) diff --git a/cmake/modules/Findrdmacm.cmake b/cmake/modules/Findrdmacm.cmake new file mode 100644 index 00000000000..310b8130e61 --- /dev/null +++ b/cmake/modules/Findrdmacm.cmake @@ -0,0 +1,18 @@ +# - Find rdma cm +# Find the rdma cm library and includes +# +# RDMACM_INCLUDE_DIR - where to find cma.h, etc. +# RDMACM_LIBRARIES - List of libraries when using rdmacm. +# RDMACM_FOUND - True if rdmacm found. + +find_path(RDMACM_INCLUDE_DIR rdma/rdma_cma.h) +find_library(RDMACM_LIBRARIES rdmacm) + +# handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE if +# all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(rdmacm DEFAULT_MSG RDMACM_LIBRARIES RDMACM_INCLUDE_DIR) + +mark_as_advanced( + RDMACM_LIBRARIES +) diff --git a/cmake/modules/Findverbs.cmake b/cmake/modules/Findverbs.cmake new file mode 100644 index 00000000000..613e3618589 --- /dev/null +++ b/cmake/modules/Findverbs.cmake @@ -0,0 +1,38 @@ +# - Find rdma verbs +# Find the rdma verbs library and includes +# +# VERBS_INCLUDE_DIR - where to find ibverbs.h, etc. +# VERBS_LIBRARIES - List of libraries when using ibverbs. +# VERBS_FOUND - True if ibverbs found. + +find_path(VERBS_INCLUDE_DIR infiniband/verbs.h) +find_library(VERBS_LIBRARIES ibverbs) + +# handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE if +# all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(verbs DEFAULT_MSG VERBS_LIBRARIES VERBS_INCLUDE_DIR) + +if(VERBS_FOUND) + message(STATUS "Found libibverbs: ${VERBS_LIBRARIES}") + include(CheckCXXSourceCompiles) + CHECK_CXX_SOURCE_COMPILES(" + #include + int main() { + struct ibv_context* ctxt; + struct ibv_exp_gid_attr gid_attr; + ibv_exp_query_gid_attr(ctxt, 1, 0, &gid_attr); + return 0; + } " HAVE_IBV_EXP) + +else() + message(STATUS "Not Found libibverbs: ${VERBS_LIBRARIES}") + if (VERBS_FIND_REQUIRED) + message(STATUS "Looked for libibverbs named ${VERBS_NAMES}.") + message(FATAL_ERROR "Could NOT find libibverbs") + endif() +endif() + +mark_as_advanced( + VERBS_LIBRARIES +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index afceda5c039..6c74f59bcd5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -297,10 +297,16 @@ if(HAVE_XIO) list(APPEND EXTRALIBS ${XIO_LIBRARY} pthread) endif(HAVE_XIO) -if(HAVE_RDMA) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${RDMA_INCLUDE_DIR}") - list(APPEND EXTRALIBS ${RDMA_LIBRARIES} pthread) -endif(HAVE_RDMA) +if(HAVE_RDMACM) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${RDMACM_INCLUDE_DIR}") + list(APPEND EXTRALIBS ${RDMACM_LIBRARIES} pthread) +endif(HAVE_RDMACM) + +if(HAVE_VERBS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${VERBS_INCLUDE_DIR}") + list(APPEND EXTRALIBS ${VERBS_LIBRARIES} pthread) +endif(HAVE_VERBS) + # sort out which allocator to use if(ALLOCATOR STREQUAL "tcmalloc") @@ -404,7 +410,7 @@ if(HAVE_XIO) endif(HAVE_XIO) set(async_rdma_common_srcs) -if(HAVE_RDMA) +if(HAVE_VERBS AND HAVE_RDMACM) list(APPEND async_rdma_common_srcs msg/async/rdma/Infiniband.cc msg/async/rdma/RDMAConnectedSocketImpl.cc @@ -412,7 +418,7 @@ if(HAVE_RDMA) msg/async/rdma/RDMAServerSocketImpl.cc msg/async/rdma/RDMAIWARPServerSocketImpl.cc msg/async/rdma/RDMAStack.cc) -endif(HAVE_RDMA) +endif() add_library(common_buffer_obj OBJECT common/buffer.cc) @@ -664,8 +670,11 @@ set(ceph_common_deps if(HAVE_UDEV) list(APPEND ceph_common_deps ${UDEV_LIBRARIES}) endif() -if(HAVE_RDMA) - list(APPEND ceph_common_deps ${RDMA_LIBRARIES}) +if(HAVE_VERBS) + list(APPEND ceph_common_deps ${VERBS_LIBRARIES}) +endif() +if(HAVE_RDMACM) + list(APPEND ceph_common_deps ${RDMACM_LIBRARIES}) endif() if(NOT WITH_SYSTEM_BOOST) list(APPEND ceph_common_deps ${ZLIB_LIBRARIES}) @@ -791,8 +800,7 @@ endif (WITH_MGR) set(librados_config_srcs librados-config.cc) add_executable(librados-config ${librados_config_srcs}) -target_link_libraries(librados-config librados global ${BLKID_LIBRARIES} ${RDMA_LIBRARIES} - ${CMAKE_DL_LIBS}) +target_link_libraries(librados-config librados global ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS}) install(TARGETS librados-config DESTINATION bin) @@ -901,7 +909,7 @@ set(ceph_osd_srcs add_executable(ceph-osd ${ceph_osd_srcs}) add_dependencies(ceph-osd erasure_code_plugins) target_link_libraries(ceph-osd osd os global-static ceph-common - ${BLKID_LIBRARIES} ${RDMA_LIBRARIES}) + ${BLKID_LIBRARIES}) if(WITH_FUSE) target_link_libraries(ceph-osd ${FUSE_LIBRARIES}) endif()