Merge pull request #28802 from lixiaoy1/cmake_pmdk

cmake: pmem/pmdk changes to cmake

Reviewed-by: Kefu Chai <kchai@redhat.com>

CMakeLists.txt

@@ -267,6 +267,17 @@ if(WITH_BLUESTORE)
   endif()
 endif()
 
+include(CMakeDependentOption)
+CMAKE_DEPENDENT_OPTION(WITH_BLUESTORE_PMEM "Enable PMDK libraries" OFF
+  "WITH_BLUESTORE" OFF)
+
+CMAKE_DEPENDENT_OPTION(WITH_SYSTEM_PMDK "Require and build with system PMDK" OFF
+  "WITH_BLUESTORE_PMEM" OFF)
+
+if(WITH_BLUESTORE_PMEM)
+  set(HAVE_BLUESTORE_PMEM ON)
+endif()
+
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "i386|i686|amd64|x86_64|AMD64|aarch64")
   option(WITH_SPDK "Enable SPDK" ON)
 else()
@@ -281,19 +292,11 @@ if(WITH_SPDK)
   set(HAVE_SPDK TRUE)
 endif(WITH_SPDK)
 
-option(WITH_PMEM "Enable PMEM" OFF)
-if(WITH_PMEM)
-  set(HAVE_PMEM ON)
-  if(NOT WITH_BLUESTORE)
-    message(SEND_ERROR "Please enable WITH_BLUESTORE for using PMEM")
-  endif()
-endif()
-
 if(WITH_BLUESTORE)
-  if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_PMEM)
+  if(NOT AIO_FOUND AND NOT HAVE_POSIXAIO AND NOT WITH_SPDK AND NOT WITH_BLUESTORE_PMEM)
     message(SEND_ERROR "WITH_BLUESTORE is ON, "
       "but none of the bluestore backends is enabled. "
-      "Please install libaio, or enable WITH_SPDK or WITH_PMEM (experimental)")
+      "Please install libaio, or enable WITH_SPDK or WITH_BLUESTORE_PMEM (experimental)")
   endif()
 endif()
 

cmake/modules/Buildpmem.cmake

@@ -0,0 +1,45 @@
+function(build_pmem)
+  include(ExternalProject)
+  set(PMDK_SRC "${CMAKE_BINARY_DIR}/src/nvml/src")
+  set(PMDK_INCLUDE "${PMDK_SRC}/include")
+
+  # Use debug PMDK libs in debug lib/rbd builds
+  if(CMAKE_BUILD_TYPE STREQUAL Debug)
+    set(PMDK_LIB_DIR "debug")
+  else()
+    set(PMDK_LIB_DIR "nondebug")
+  endif()
+  set(PMDK_LIB "${PMDK_SRC}/${PMDK_LIB_DIR}")
+
+  ExternalProject_Add(nvml_ext
+      GIT_REPOSITORY "https://github.com/ceph/nvml.git"
+      GIT_TAG "dd622819dd4ee97d3920f913c70be"
+      SOURCE_DIR ${CMAKE_BINARY_DIR}/src/nvml
+      CONFIGURE_COMMAND ""
+      # Explicitly built w/o NDCTL, otherwise if ndtcl is present on the
+      # build system tests statically linking to librbd (which uses
+      # libpmemobj) will not link (because we don't build the ndctl
+      # static library here).
+      BUILD_COMMAND $(MAKE) NDCTL_ENABLE=n
+      BUILD_IN_SOURCE 1
+      BUILD_BYPRODUCTS "${PMDK_LIB}/libpmem.a" "${PMDK_LIB}/libpmemobj.a"
+      INSTALL_COMMAND "true")
+
+    # libpmem
+    add_library(pmem::pmem STATIC IMPORTED)
+    add_dependencies(pmem::pmem nvml_ext)
+    file(MAKE_DIRECTORY ${PMDK_INCLUDE})
+    set_target_properties(pmem::pmem PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES ${PMDK_INCLUDE}
+      IMPORTED_LOCATION "${PMDK_LIB}/libpmem.a"
+      INTERFACE_LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+
+    # libpmemobj
+    add_library(pmem::pmemobj STATIC IMPORTED)
+    add_dependencies(pmem::pmemobj nvml_ext)
+    set_target_properties(pmem::pmemobj PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES ${PMDK_INCLUDE}
+      IMPORTED_LOCATION "${PMDK_LIB}/libpmemobj.a"
+      INTERFACE_LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+
+endfunction()

cmake/modules/Findpmem.cmake

@@ -0,0 +1,45 @@
+# - Find pmem
+#
+# PMEM_INCLUDE_DIR - Where to find libpmem.h
+# PMEM_LIBRARIES - List of libraries when using pmdk.
+# pmem_FOUND - True if pmem found.
+# PMEMOBJ_INCLUDE_DIR - Where to find libpmemobj.h
+# PMEMOBJ_LIBRARIES - List of libraries when using pmdk obj.
+# pmemobj_FOUND - True if pmemobj found.
+
+find_path(PMEM_INCLUDE_DIR libpmem.h)
+find_library(PMEM_LIBRARIES pmem)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(pmem
+  DEFAULT_MSG PMEM_LIBRARIES PMEM_INCLUDE_DIR)
+
+mark_as_advanced(
+  PMEM_INCLUDE_DIR
+  PMEM_LIBRARIES)
+
+if(pmem_FOUND AND NOT TARGET pmem::pmem)
+  add_library(pmem::pmem UNKNOWN IMPORTED)
+  set_target_properties(pmem::pmem PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${PMEM_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${PMEM_LIBRARIES}")
+endif()
+
+find_path(PMEMOBJ_INCLUDE_DIR libpmemobj.h)
+find_library(PMEMOBJ_LIBRARIES pmemobj)
+
+find_package_handle_standard_args(pmemobj
+  DEFAULT_MSG PMEMOBJ_LIBRARIES PMEMOBJ_INCLUDE_DIR)
+
+mark_as_advanced(
+  PMEMOBJ_INCLUDE_DIR
+  PMEMOBJ_LIBRARIES)
+
+if(pmemobj_FOUND AND NOT TARGET pmem::pmemobj)
+  add_library(pmem::pmemobj UNKNOWN IMPORTED)
+  set_target_properties(pmem::pmemobj PROPERTIES
+    INTERFACE_INCLUDE_DIRECTORIES "${PMEMOBJ_INCLUDE_DIR}"
+    IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+    IMPORTED_LOCATION "${PMEMOBJ_LIBRARIES}")
+endif()

src/CMakeLists.txt

@@ -401,6 +401,15 @@ if(WITH_DPDK)
   list(APPEND ceph_common_deps common_async_dpdk)
 endif()
 
+if(WITH_BLUESTORE_PMEM)
+  if(WITH_SYSTEM_PMDK)
+    find_package(pmem REQUIRED)
+  else()
+    include(Buildpmem)
+    build_pmem()
+  endif()
+endif()
+
 add_library(common STATIC ${ceph_common_objs})
 target_link_libraries(common ${ceph_common_deps})
 

src/include/config-h.in.cmake

@@ -111,8 +111,8 @@
 /* DPDK conditional compilation */
 #cmakedefine HAVE_DPDK
 
-/* PMEM conditional compilation */
-#cmakedefine HAVE_PMEM
+/* PMEM_DEVICE (OSD) conditional compilation */
+#cmakedefine HAVE_BLUESTORE_PMEM
 
 /* Defined if LevelDB supports bloom filters */
 #cmakedefine HAVE_LEVELDB_FILTER_POLICY

src/os/CMakeLists.txt

@@ -46,10 +46,10 @@ if(WITH_FUSE)
     FuseStore.cc)
 endif(WITH_FUSE)
 
-if(WITH_PMEM)
+if(WITH_BLUESTORE_PMEM)
   list(APPEND libos_srcs
     bluestore/PMEMDevice.cc)
-endif(WITH_PMEM)
+endif()
 
 if(HAVE_LIBXFS)
   list(APPEND libos_srcs
@@ -121,31 +121,9 @@ if(WITH_BLUESTORE)
     DESTINATION bin)
 endif()
 
-if(WITH_PMEM)
-  include(ExternalProject)
-  ExternalProject_Add(nvml_ext
-    DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/src/
-    GIT_REPOSITORY "https://github.com/ceph/nvml.git"
-    GIT_TAG "dd622819dd4ee97d3920f913c70be"
-    SOURCE_DIR ${CMAKE_BINARY_DIR}/src/nvml
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND $(MAKE)
-    BUILD_IN_SOURCE 1
-    INSTALL_COMMAND "true")
-
-  ExternalProject_Add_Step(nvml_ext forcebuild
-    DEPENDEES configure
-    DEPENDERS build
-    COMMAND "true"
-    ALWAYS 1)
-  add_library(pmem STATIC IMPORTED GLOBAL)
-  add_dependencies(pmem nvml_ext)
-  set_target_properties(pmem PROPERTIES
-    IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/src/nvml/src/nondebug/libpmem.a"
-    INTERFACE_LINK_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
-  target_link_libraries(os pmem)
-  target_include_directories(os SYSTEM PRIVATE "${CMAKE_BINARY_DIR}/src/nvml/src/include")
-endif(WITH_PMEM)
+if(WITH_BLUESTORE_PMEM)
+  target_link_libraries(os pmem::pmem)
+endif()
 
 if(WITH_LTTNG AND WITH_EVENTTRACE)
   add_dependencies(os eventtrace_tp)

src/os/bluestore/BlockDevice.cc

@@ -27,7 +27,7 @@
 #include "NVMEDevice.h"
 #endif
 
-#if defined(HAVE_PMEM)
+#if defined(HAVE_BLUESTORE_PMEM)
 #include "PMEMDevice.h"
 #include "libpmem.h"
 #endif
@@ -96,7 +96,7 @@ BlockDevice *BlockDevice::create(CephContext* cct, const string& path,
       type = "ust-nvme";
   }
 
-#if defined(HAVE_PMEM)
+#if defined(HAVE_BLUESTORE_PMEM)
   if (type == "kernel") {
     int is_pmem = 0;
     size_t map_len = 0;
@@ -115,7 +115,7 @@ BlockDevice *BlockDevice::create(CephContext* cct, const string& path,
 
   dout(1) << __func__ << " path " << path << " type " << type << dendl;
 
-#if defined(HAVE_PMEM)
+#if defined(HAVE_BLUESTORE_PMEM)
   if (type == "pmem") {
     return new PMEMDevice(cct, cb, cbpriv);
   }