src: extract backend driver from bluestore to access device

1. Both bluestore or other component e.g. rbd could use the same
   driver to access storage device. It's better to use one library
   to integrate the driver.
2. os and crimson-alienstore are static libraries. Link blk library into
   them.

Main changes are below:
1. move backend driver into src/blk
   src/$ mkdir -p blk/{aio,kernel,spdk,pmem,zns}
   src/$ mv os/bluestore/{BlockDevice.h,BlockDevice.cc} blk
   src/$ mv os/bluestore/{ceph_aio.h} aio/aio.h
   src/$ mv os/bluestore/{aio.cc} aio/
   src/$ mv os/bluestore/{KernelDevice.h,KernelDevice.cc} kernel/
   src/$ mv os/bluestore/{ceph_io_uring.h} kernel/io_uring.h
   src/$ mv os/bluestore/{io_uring.cc} kernel_drv/
   src/$ mv os/bluestore/{NVMEDevice.h,NVMEDevice.cc} spdk/
   src/$ mv os/bluestore/{PMEMDevice.h,PMEMDevice.cc} pmem/
   src/$ mv os/bluestore/{HMSMRDevice.h,HMSMRDevice.cc} zns/
2. customize macro name in header file to remove bluestore specific text
3. adjust header file patch in source code
4. create cmake rule blk/CMakeLists.txt to build blk
5. modify src/CMakeLists.txt to integrate blk
6. modify other CMakeLists.txt to adapt to new file structure.

Signed-off-by: Changcheng Liu <changcheng.liu@aliyun.com>
Co-authored-by: Kefu Chai <kefu@redhat.com>

src/CMakeLists.txt

@@ -544,6 +544,7 @@ endif()
 
 add_subdirectory(kv)
 add_subdirectory(os)
+add_subdirectory(blk)
 
 add_subdirectory(osd)
 

src/blk/BlockDevice.cc

@@ -20,20 +20,20 @@
 #include "BlockDevice.h"
 
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
-#include "KernelDevice.h"
+#include "kernel/KernelDevice.h"
 #endif
 
 #if defined(HAVE_SPDK)
-#include "NVMEDevice.h"
+#include "spdk/NVMEDevice.h"
 #endif
 
 #if defined(HAVE_BLUESTORE_PMEM)
-#include "PMEMDevice.h"
+#include "pmem/PMEMDevice.h"
 #include "libpmem.h"
 #endif
 
 #if defined(HAVE_LIBZBC)
-#include "HMSMRDevice.h"
+#include "zns/HMSMRDevice.h"
 extern "C" {
 #include <libzbc/zbc.h>
 }

src/blk/BlockDevice.h

@@ -14,8 +14,8 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_BLOCKDEVICE_H
-#define CEPH_OS_BLUESTORE_BLOCKDEVICE_H
+#ifndef CEPH_BLK_BLOCKDEVICE_H
+#define CEPH_BLK_BLOCKDEVICE_H
 
 #include <atomic>
 #include <condition_variable>
@@ -31,7 +31,7 @@
 #include "include/common_fwd.h"
 
 #if defined(HAVE_LIBAIO) || defined(HAVE_POSIXAIO)
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #endif
 #include "include/ceph_assert.h"
 #include "include/buffer.h"
@@ -261,4 +261,4 @@ protected:
   }
 };
 
-#endif //CEPH_OS_BLUESTORE_BLOCKDEVICE_H
+#endif //CEPH_BLK_BLOCKDEVICE_H

src/blk/CMakeLists.txt

@@ -0,0 +1,61 @@
+if(WITH_BLUESTORE OR WITH_RBD_RWL)
+list(APPEND libblk_srcs
+  BlockDevice.cc)
+endif()
+
+if(HAVE_LIBAIO OR HAVE_POSIXAIO)
+  list(APPEND libblk_srcs
+    kernel/KernelDevice.cc
+    kernel/io_uring.cc
+    aio/aio.cc)
+endif()
+
+if(WITH_BLUESTORE_PMEM)
+  list(APPEND libblk_srcs
+    pmem/PMEMDevice.cc)
+endif()
+
+if(WITH_SPDK)
+  list(APPEND libblk_srcs
+    spdk/NVMEDevice.cc)
+endif()
+
+if(HAVE_LIBZBC)
+  list(APPEND libblk_srcs
+    zns/HMSMRDevice.cc)
+endif()
+
+add_library(blk ${libblk_srcs})
+target_include_directories(blk PRIVATE "./")
+
+if(HAVE_LIBAIO)
+  target_link_libraries(blk PUBLIC ${AIO_LIBRARIES})
+endif(HAVE_LIBAIO)
+
+if(WITH_SPDK)
+  target_link_libraries(blk PRIVATE ${SPDK_LIBRARIES})
+endif()
+
+if(HAVE_LIBZBC)
+  target_link_libraries(blk ${ZBC_LIBRARIES})
+endif()
+
+if(WITH_BLUESTORE_PMEM OR WITH_RBD_RWL)
+  target_link_libraries(blk
+    PUBLIC pmem::pmemobj
+    PRIVATE pmem::pmem)
+endif()
+
+if(WITH_EVENTTRACE)
+  add_dependencies(blk eventtrace_tp)
+endif()
+
+if(WITH_LIBURING)
+  if(WITH_SYSTEM_LIBURING)
+    find_package(uring REQUIRED)
+  else()
+    include(Builduring)
+    build_uring()
+  endif()
+  target_link_libraries(blk PRIVATE uring::uring)
+endif()

src/blk/aio/aio.cc

@@ -2,7 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 #include <algorithm>
-#include "ceph_aio.h"
+#include "aio.h"
 
 std::ostream& operator<<(std::ostream& os, const aio_t& aio)
 {

src/blk/kernel/KernelDevice.cc

@@ -33,7 +33,7 @@
 #include "common/numa.h"
 
 #include "global/global_context.h"
-#include "ceph_io_uring.h"
+#include "io_uring.h"
 
 #define dout_context cct
 #define dout_subsys ceph_subsys_bdev

src/blk/kernel/KernelDevice.h

@@ -12,8 +12,8 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_KERNELDEVICE_H
-#define CEPH_OS_BLUESTORE_KERNELDEVICE_H
+#ifndef CEPH_BLK_KERNELDEVICE_H
+#define CEPH_BLK_KERNELDEVICE_H
 
 #include <atomic>
 
@@ -22,7 +22,7 @@
 #include "common/Thread.h"
 #include "include/utime.h"
 
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #include "BlockDevice.h"
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)

src/blk/kernel/io_uring.cc

@@ -1,7 +1,7 @@
 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
 
-#include "ceph_io_uring.h"
+#include "io_uring.h"
 
 #if defined(HAVE_LIBURING) && defined(__x86_64__)
 

src/blk/kernel/io_uring.h

@@ -6,7 +6,7 @@
 #include "acconfig.h"
 
 #include "include/types.h"
-#include "ceph_aio.h"
+#include "aio/aio.h"
 
 struct ioring_data;
 

src/blk/pmem/PMEMDevice.h

@@ -14,14 +14,14 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_PMEMDEVICE_H
-#define CEPH_OS_BLUESTORE_PMEMDEVICE_H
+#ifndef CEPH_BLK_PMEMDEVICE_H
+#define CEPH_BLK_PMEMDEVICE_H
 
 #include <atomic>
 
 #include "os/fs/FS.h"
 #include "include/interval_set.h"
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #include "BlockDevice.h"
 
 class PMEMDevice : public BlockDevice {

src/blk/spdk/NVMEDevice.h

@@ -14,8 +14,8 @@
  *
  */
 
-#ifndef CEPH_OS_BLUESTORE_NVMEDEVICE
-#define CEPH_OS_BLUESTORE_NVMEDEVICE
+#ifndef CEPH_BLK_NVMEDEVICE
+#define CEPH_BLK_NVMEDEVICE
 
 #include <queue>
 #include <map>

src/blk/zns/HMSMRDevice.cc

@@ -35,7 +35,7 @@
 #include "common/numa.h"
 
 #include "global/global_context.h"
-#include "ceph_io_uring.h"
+#include "kernel/io_uring.h"
 
 extern "C" {
 #include <libzbc/zbc.h>

src/blk/zns/HMSMRDevice.h

@@ -16,8 +16,8 @@
 // Copied from KernelDevice with HM-SMR specific functionality added.  Will be
 // further specialized for HM-SMR.
 
-#ifndef CEPH_OS_BLUESTORE_HMSMRDEVICE_H
-#define CEPH_OS_BLUESTORE_HMSMRDEVICE_H
+#ifndef CEPH_BLK_HMSMRDEVICE_H
+#define CEPH_BLK_HMSMRDEVICE_H
 
 #include <atomic>
 
@@ -26,7 +26,7 @@
 #include "common/Thread.h"
 #include "include/utime.h"
 
-#include "ceph_aio.h"
+#include "aio/aio.h"
 #include "BlockDevice.h"
 
 #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK)
@@ -159,4 +159,4 @@ public:
   void close() final;
 };
 
-#endif
+#endif //CEPH_BLK_HMSMRDEVICE_H

src/crimson/os/alienstore/CMakeLists.txt

@@ -28,7 +28,6 @@ list(APPEND crimson_alien_srcs
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/Allocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/AvlAllocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapFreelistManager.cc
-  ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlockDevice.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueFS.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/bluefs_types.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BlueRocksEnv.cc
@@ -37,23 +36,13 @@ list(APPEND crimson_alien_srcs
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/fastbmap_allocator_impl.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/FreelistManager.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/HybridAllocator.cc
-  ${PROJECT_SOURCE_DIR}/src/os/bluestore/io_uring.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/StupidAllocator.cc
   ${PROJECT_SOURCE_DIR}/src/os/bluestore/BitmapAllocator.cc)
 
-if(HAVE_LIBAIO OR HAVE_POSIXAIO)
-  list(APPEND crimson_alien_srcs
-     ${PROJECT_SOURCE_DIR}/src/os/bluestore/KernelDevice.cc
-     ${PROJECT_SOURCE_DIR}/src/os/bluestore/aio.cc)
-endif()
-
 add_library(crimson-alienstore STATIC ${crimson_alien_srcs}
   $<TARGET_OBJECTS:compressor_objs>
   $<TARGET_OBJECTS:crush_objs>
   $<TARGET_OBJECTS:common_prioritycache_obj>)
-if(HAVE_LIBAIO)
-  target_link_libraries(crimson-alienstore ${AIO_LIBRARIES})
-endif(HAVE_LIBAIO)
 
 target_compile_definitions(crimson-alienstore PRIVATE -DWITH_SEASTAR -DWITH_ALIEN)
 target_include_directories(crimson-alienstore PRIVATE
@@ -64,3 +53,4 @@ target_link_libraries(crimson-alienstore heap_profiler)
 target_link_libraries(crimson-alienstore ${BLKID_LIBRARIES})
 target_link_libraries(crimson-alienstore ${UDEV_LIBRARIES})
 target_link_libraries(crimson-alienstore crimson)
+target_link_libraries(crimson-alienstore blk)

src/librbd/CMakeLists.txt

@@ -217,8 +217,7 @@ target_link_libraries(rbd_internal PRIVATE
 
 if(WITH_RBD_RWL)
   target_link_libraries(rbd_internal
-    PUBLIC pmem::pmemobj
-    PRIVATE pmem::pmem)
+    PUBLIC blk)
 endif()
 
 add_library(librbd ${CEPH_SHARED}

src/os/CMakeLists.txt

@@ -23,7 +23,6 @@ if(WITH_BLUESTORE)
   list(APPEND libos_srcs
     bluestore/Allocator.cc
     bluestore/BitmapFreelistManager.cc
-    bluestore/BlockDevice.cc
     bluestore/BlueFS.cc
     bluestore/bluefs_types.cc
     bluestore/BlueRocksEnv.cc
@@ -35,19 +34,11 @@ if(WITH_BLUESTORE)
     bluestore/BitmapAllocator.cc
     bluestore/AvlAllocator.cc
     bluestore/HybridAllocator.cc
-    bluestore/io_uring.cc
   )
 endif(WITH_BLUESTORE)
 
-if(HAVE_LIBAIO OR HAVE_POSIXAIO)
-  list(APPEND libos_srcs
-    bluestore/KernelDevice.cc
-    bluestore/aio.cc)
-endif()
-
 if(HAVE_LIBZBC)
   list(APPEND libos_srcs
-    bluestore/HMSMRDevice.cc
     bluestore/ZonedAllocator.cc)
 endif()
 
@@ -56,11 +47,6 @@ if(WITH_FUSE)
     FuseStore.cc)
 endif(WITH_FUSE)
 
-if(WITH_BLUESTORE_PMEM)
-  list(APPEND libos_srcs
-    bluestore/PMEMDevice.cc)
-endif()
-
 if(HAVE_LIBXFS)
   list(APPEND libos_srcs
     filestore/XfsFileStoreBackend.cc
@@ -76,12 +62,8 @@ if(HAVE_LIBZFS)
   list(APPEND libos_srcs $<TARGET_OBJECTS:os_zfs_objs>)
 endif()
 
-if(WITH_SPDK)
-  list(APPEND libos_srcs
-    bluestore/NVMEDevice.cc)
-endif()
-
 add_library(os STATIC ${libos_srcs})
+target_link_libraries(os blk)
 
 target_link_libraries(os heap_profiler kv)
 
@@ -94,14 +76,6 @@ if(WITH_BLUEFS)
   install(TARGETS bluefs DESTINATION lib)
 endif(WITH_BLUEFS)
 
-if(HAVE_LIBAIO)
-  target_link_libraries(os ${AIO_LIBRARIES})
-endif(HAVE_LIBAIO)
-
-if(HAVE_LIBZBC)
-  target_link_libraries(os ${ZBC_LIBRARIES})
-endif()
-
 if(WITH_FUSE)
   target_link_libraries(os FUSE::FUSE)
 endif()
@@ -110,11 +84,6 @@ if(HAVE_LIBZFS)
   target_link_libraries(os ${ZFS_LIBRARIES})
 endif()
 
-if(WITH_SPDK)
-  target_link_libraries(os
-    ${SPDK_LIBRARIES})
-endif()
-
 if(WITH_LTTNG)
   add_dependencies(os objectstore-tp)
   add_dependencies(os bluestore-tp)
@@ -134,21 +103,3 @@ if(WITH_BLUESTORE)
   install(TARGETS ceph-bluestore-tool
     DESTINATION bin)
 endif()
-
-if(WITH_BLUESTORE_PMEM)
-  target_link_libraries(os pmem::pmem)
-endif()
-
-if(WITH_EVENTTRACE)
-  add_dependencies(os eventtrace_tp)
-endif()
-
-if(WITH_LIBURING)
-  if(WITH_SYSTEM_LIBURING)
-    find_package(uring REQUIRED)
-  else()
-    include(Builduring)
-    build_uring()
-  endif()
-  target_link_libraries(os uring::uring)
-endif()

src/os/bluestore/BlueFS.cc

@@ -8,7 +8,6 @@
 #include "common/debug.h"
 #include "common/errno.h"
 #include "common/perf_counters.h"
-#include "BlockDevice.h"
 #include "Allocator.h"
 #include "include/ceph_assert.h"
 #include "common/admin_socket.h"

src/os/bluestore/BlueFS.h

@@ -7,7 +7,7 @@
 #include <mutex>
 
 #include "bluefs_types.h"
-#include "BlockDevice.h"
+#include "blk/BlockDevice.h"
 
 #include "common/RefCountedObj.h"
 #include "common/ceph_context.h"

src/os/bluestore/BlueStore.h

@@ -49,7 +49,6 @@
 #include "os/ObjectStore.h"
 
 #include "bluestore_types.h"
-#include "BlockDevice.h"
 #include "BlueFS.h"
 #include "common/EventTrace.h"
 

src/test/objectstore/test_bdev.cc

@@ -12,7 +12,7 @@
 #include "include/stringify.h"
 #include "common/errno.h"
 
-#include "os/bluestore/BlockDevice.h"
+#include "blk/BlockDevice.h"
 
 class TempBdev {
 public: