Merge pull request #14723 from neha-ojha/objclass-sdk

osd: objclass sdk

Reviewed-by: Josh Durgin <jdurgin@redhat.com>
Reviewed-by: Sage Weil <sage@redhat.com>
Reviewed-by: Kefu Chai <kchai@redhat.com>

ceph.spec.in

@@ -718,6 +718,15 @@ This package contains the Java libraries for the Ceph File System.
 
 %endif
 
+%package -n rados-objclass-devel
+Summary:        RADOS object class development kit
+Group:          Development/Libraries
+License:        LGPL-2.0
+Requires:       librados2-devel = %{epoch}:%{version}-%{release}
+%description -n rados-objclass-devel
+This package contains libraries and headers needed to develop RADOS object
+class plugins.
+
 %if 0%{with selinux}
 
 %package selinux
@@ -1650,6 +1659,11 @@ ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1
 %{_javadir}/libcephfs-test.jar
 %endif
 
+%files -n rados-objclass-devel
+%defattr(-,root,root,-)
+%dir %{_includedir}/rados
+%{_includedir}/rados/objclass.h
+
 %if 0%{with selinux}
 %files selinux
 %defattr(-,root,root,-)

debian/control

@@ -825,3 +825,11 @@ Section: java
 Depends: libcephfs2 (= ${binary:Version}), ${java:Depends},
          ${misc:Depends}, ${shlibs:Depends}
 Description: Java Native Interface library for CephFS Java bindings
+
+Package: rados-objclass-dev
+Architecture: linux-any
+Section: libdevel
+Depends: librados-dev (= ${binary:Version}) ${misc:Depends}
+Description: RADOS object class development kit.
+ .
+ This package contains development files needed for building RADOS object class plugins.

debian/rados-objclass-dev.install

@@ -0,0 +1 @@
+usr/include/rados/objclass.h

doc/rados/api/index.rst

@@ -18,4 +18,5 @@ Ceph, your own interface to Ceph, etc.).
    Introduction to librados <librados-intro>
    librados (C) <librados>
    librados (C++) <libradospp>
-   librados (Python) <python>
+   librados (Python) <python>
+   object class <objclass-sdk>

doc/rados/api/objclass-sdk.rst

@@ -0,0 +1,37 @@
+===========================
+SDK for Ceph Object Classes
+===========================
+
+`Ceph` can be extended by creating shared object classes called `Ceph Object 
+Classes`. The existing framework to build these object classes has dependencies 
+on the internal functionality of `Ceph`, which restricts users to build object 
+classes within the tree. The aim of this project is to create an independent 
+object class interface, which can be used to build object classes outside the 
+`Ceph` tree. This allows us to have two types of object classes, 1) those that 
+have in-tree dependencies and reside in the tree and 2) those that can make use 
+of the `Ceph Object Class SDK framework` and can be built outside of the `Ceph` 
+tree because they do not depend on any internal implementation of `Ceph`. This 
+project decouples object class development from Ceph and encourages creation 
+and distribution of object classes as packages.
+
+In order to demonstrate the use of this framework, we have provided an example 
+called ``cls_sdk``, which is a very simple object class that makes use of the 
+SDK framework. This object class resides in the ``src/cls`` directory. 
+
+Installing objclass.h
+---------------------
+
+The object class interface that enables out-of-tree development of object 
+classes resides in ``src/include/rados/`` and gets installed with `Ceph` 
+installation. After running ``make install``, you should be able to see it 
+in ``<prefix>/include/rados``. ::
+
+        ls /usr/local/include/rados
+
+Using the SDK example
+---------------------
+
+The ``cls_sdk`` object class resides in ``src/cls/sdk/``. This gets built and 
+loaded into Ceph, with the Ceph build process. You can run the 
+``ceph_test_cls_sdk`` unittest, which resides in ``src/test/cls_sdk/``, 
+to test this class.

qa/suites/rados/basic/tasks/rados_cls_all.yaml

@@ -1,3 +1,11 @@
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd_class_load_list: "cephfs hello journal lock log numops rbd refcount 
+                              replica_log rgw sdk statelog timeindex user version"
+        osd_class_default_list: "cephfs hello journal lock log numops rbd refcount 
+                                 replica_log rgw sdk statelog timeindex user version"
 tasks:
 - install:
 - ceph:

qa/workunits/cls/test_cls_sdk.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -e
+
+ceph_test_cls_sdk
+
+exit 0

src/cls/CMakeLists.txt

@@ -3,6 +3,11 @@
 set(cls_dir ${CMAKE_INSTALL_LIBDIR}/rados-classes)
 set(cls_embedded_srcs)
 
+# cls_sdk
+add_library(cls_sdk SHARED sdk/cls_sdk.cc)
+set_target_properties(cls_sdk PROPERTIES VERSION "1.0.0" SOVERSION "1")
+install(TARGETS cls_sdk DESTINATION ${cls_dir})
+
 # cls_hello
 set(cls_hello_srcs hello/cls_hello.cc)
 add_library(cls_hello SHARED ${cls_hello_srcs})

src/cls/sdk/cls_sdk.cc

@@ -0,0 +1,131 @@
+/*
+ * This is an example RADOS object class built using only the Ceph SDK interface.
+ */
+#include "include/rados/objclass.h"
+
+CLS_VER(1,0)
+CLS_NAME(sdk)
+
+cls_handle_t h_class;
+cls_method_handle_t h_test_coverage_write;
+cls_method_handle_t h_test_coverage_replay;
+
+/**
+ * test_coverage_write - a "write" method that creates an object
+ *
+ * This method modifies the object by making multiple write calls (write,
+ * setxattr and set_val).
+ */
+static int test_coverage_write(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+  // create the object
+  int ret = cls_cxx_create(hctx, false);
+  if (ret < 0) {
+    CLS_LOG(0, "ERROR: %s(): cls_cxx_create returned %d", __func__, ret);	  
+    return ret;
+  }
+
+  uint64_t size;
+  // get the size of the object
+  ret = cls_cxx_stat(hctx, &size, NULL);
+  if (ret < 0)
+    return ret;
+  
+  std::string c = "test";
+  bufferlist bl;
+  bl.append(c);
+
+  // write to the object
+  ret = cls_cxx_write(hctx, 0, bl.length(), &bl);
+  if (ret < 0)
+    return ret;
+  
+  uint64_t new_size;
+  // get the new size of the object
+  ret = cls_cxx_stat(hctx, &new_size, NULL);
+  if (ret < 0)
+    return ret;
+
+  // make some change to the xattr
+  ret = cls_cxx_setxattr(hctx, "foo", &bl);
+  if (ret < 0)
+    return ret;
+
+  // make some change to the omap
+  ret = cls_cxx_map_set_val(hctx, "foo", &bl);
+  if (ret < 0)
+    return ret;
+
+  return 0;
+}
+
+/**
+ * test_coverage_replay - a "read" method to retrieve previously written data
+ *
+ * This method reads the object by making multiple read calls (read, getxattr 
+ * and get_val). It also removes the object after reading.
+ */
+
+static int test_coverage_replay(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+  CLS_LOG(0, "reading already written object");
+  uint64_t size;
+  // get the size of the object
+  int ret = cls_cxx_stat(hctx, &size, NULL);
+  if (ret < 0)
+    return ret;
+  
+  bufferlist bl;
+  // read the object entry
+  ret = cls_cxx_read(hctx, 0, size, &bl);
+  if (ret < 0)
+    return ret;
+
+  // if the size is incorrect 
+  if (bl.length() != size)
+    return -EIO;
+  
+  bl.clear();
+
+  // read xattr entry
+  ret = cls_cxx_getxattr(hctx, "foo", &bl);
+  if (ret < 0)
+    return ret;
+
+  // if the size is incorrect
+  if (bl.length() != size)
+    return -EIO;
+  
+  bl.clear();
+
+  // read omap entry
+  ret = cls_cxx_map_get_val(hctx, "foo", &bl);
+  if (ret < 0)
+    return ret;
+
+  // if the size is incorrect
+  if (bl.length() != size)
+    return -EIO;
+
+  // remove the object
+  ret = cls_cxx_remove(hctx);
+  if (ret < 0)
+    return ret;
+  
+  return 0;
+}
+
+void __cls_init()
+{
+  CLS_LOG(0, "loading cls_sdk");
+
+  cls_register("sdk", &h_class);
+
+  cls_register_cxx_method(h_class, "test_coverage_write",
+      CLS_METHOD_RD|CLS_METHOD_WR,
+      test_coverage_write, &h_test_coverage_write);
+  
+  cls_register_cxx_method(h_class, "test_coverage_replay",
+      CLS_METHOD_RD|CLS_METHOD_WR,
+      test_coverage_replay, &h_test_coverage_replay);
+}

src/include/CMakeLists.txt

@@ -8,6 +8,7 @@ install(FILES rados/librados.h
   memory.h
   page.h
   crc32c.h
+  rados/objclass.h
   DESTINATION include/rados)
 install(FILES
   radosstriper/libradosstriper.h

src/include/rados/objclass.h

@@ -0,0 +1,181 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_OBJCLASS_OBJCLASS_PUBLIC_H
+#define CEPH_OBJCLASS_OBJCLASS_PUBLIC_H
+
+#ifdef __cplusplus
+
+#include "buffer.h"
+
+extern "C" {
+#endif
+
+#ifndef BUILDING_FOR_EMBEDDED
+#define CLS_VER(maj,min) \
+int __cls_ver__## maj ## _ ##min = 0; \
+int __cls_ver_maj = maj; \
+int __cls_ver_min = min;
+
+#define CLS_NAME(name) \
+int __cls_name__## name = 0; \
+const char *__cls_name = #name;
+#define CLS_INIT(name) \
+void CEPH_CLS_API __cls_init()
+#else
+#define CLS_VER(maj,min)
+#define CLS_NAME(name)
+#define CLS_INIT(name) \
+void CEPH_CLS_API name##_cls_init()
+#endif
+
+#define CLS_METHOD_RD       0x1 /// method executes read operations
+#define CLS_METHOD_WR       0x2 /// method executes write operations
+#define CLS_METHOD_PROMOTE  0x8 /// method cannot be proxied to base tier
+
+#define CLS_LOG(level, fmt, ...)                                        \
+  cls_log(level, "<cls> %s:%d: " fmt, __FILE__, __LINE__, ##__VA_ARGS__)
+#define CLS_ERR(fmt, ...) CLS_LOG(0, fmt, ##__VA_ARGS__)
+
+/**
+ * Initialize a class.
+ */
+void __cls_init();
+
+/**
+ * @typdef cls_handle_t
+ *
+ * A handle for interacting with the object class.
+ */
+typedef void *cls_handle_t;
+
+/**
+ * @typedef cls_method_handle_t
+ *
+ * A handle for interacting with the method of the object class.
+ */
+typedef void *cls_method_handle_t;
+
+/**
+ * @typedef cls_method_context_t
+ *
+ * A context for the method of the object class.
+ */
+typedef void* cls_method_context_t;
+
+/*class utils*/
+extern int cls_log(int level, const char *format, ...)
+  __attribute__((__format__(printf, 2, 3)));
+
+/* class registration api */
+extern int cls_register(const char *name, cls_handle_t *handle);
+
+#ifdef __cplusplus
+}
+
+/**
+ * @typedef cls_method_cxx_call_t
+ *
+ */
+typedef int (*cls_method_cxx_call_t)(cls_method_context_t ctx,
+    class buffer::list *inbl, class buffer::list *outbl);
+
+/**
+ * Register a method.
+ *
+ * @param hclass
+ * @param method
+ * @param flags
+ * @param class_call
+ * @param handle
+ */
+extern int cls_register_cxx_method(cls_handle_t hclass, const char *method, int flags,
+                                   cls_method_cxx_call_t class_call, cls_method_handle_t *handle);
+
+/**
+ * Create an object.
+ *
+ * @param hctx
+ * @param exclusive
+ */
+extern int cls_cxx_create(cls_method_context_t hctx, bool exclusive);
+
+/**
+ * Remove an object.
+ *
+ * @param hctx
+ */
+extern int cls_cxx_remove(cls_method_context_t hctx);
+
+/**
+ * Check on the status of an object.
+ *
+ * @param hctx
+ * @param size
+ * @param mtime
+ */
+extern int cls_cxx_stat(cls_method_context_t hctx, uint64_t *size, time_t *mtime);
+
+/**
+ * Read contents of an object.
+ *
+ * @param hctx
+ * @param ofs
+ * @param len
+ * @param bl
+ */
+extern int cls_cxx_read(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
+
+/**
+ * Write to the object.
+ *
+ * @param hctx
+ * @param ofs
+ * @param len
+ * @param bl
+ */
+extern int cls_cxx_write(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
+
+/**
+ * Get xattr of the object.
+ *
+ * @param hctx
+ * @param name
+ * @param outbl
+ */
+extern int cls_cxx_getxattr(cls_method_context_t hctx, const char *name,
+                            bufferlist *outbl);
+
+/**
+ * Set xattr of the object.
+ *
+ * @param hctx
+ * @param name
+ * @param inbl
+ */
+extern int cls_cxx_setxattr(cls_method_context_t hctx, const char *name,
+                            bufferlist *inbl);
+
+/**
+ * Get value corresponding to a key from the map.
+ *
+ * @param hctx
+ * @param key
+ * @param outbl
+ */
+extern int cls_cxx_map_get_val(cls_method_context_t hctx,
+                               const std::string &key, bufferlist *outbl);
+
+/**
+ * Set value corresponding to a key in the map.
+ *
+ * @param hctx
+ * @param key
+ * @param inbl
+ */
+extern int cls_cxx_map_set_val(cls_method_context_t hctx,
+                               const std::string &key, bufferlist *inbl);
+
+#endif
+
+#endif

src/objclass/objclass.h

@@ -10,6 +10,7 @@
 #include "msg/msg_types.h"
 #include "common/hobject.h"
 #include "common/ceph_time.h"
+#include "include/rados/objclass.h"
 
 struct obj_list_watch_response_t;
 
@@ -22,40 +23,9 @@ struct obj_list_watch_response_t;
 extern "C" {
 #endif
 
-#ifndef BUILDING_FOR_EMBEDDED
-#define CLS_VER(maj,min) \
-int __cls_ver__## maj ## _ ##min = 0; \
-int __cls_ver_maj = maj; \
-int __cls_ver_min = min;
-
-#define CLS_NAME(name) \
-int __cls_name__## name = 0; \
-const char *__cls_name = #name;
-#define CLS_INIT(name) \
-void CEPH_CLS_API __cls_init()
-#else
-#define CLS_VER(maj,min)
-#define CLS_NAME(name)
-#define CLS_INIT(name) \
-void CEPH_CLS_API name##_cls_init()
-#endif
-
-#define CLS_METHOD_RD       0x1 /// method executes read operations
-#define CLS_METHOD_WR       0x2 /// method executes write operations
 #define CLS_METHOD_PUBLIC   0x4 /// unused
-#define CLS_METHOD_PROMOTE  0x8 /// method cannot be proxied to base tier
 
-
-#define CLS_LOG(level, fmt, ...)					\
-  cls_log(level, "<cls> %s:%d: " fmt, __FILE__, __LINE__, ##__VA_ARGS__)
-#define CLS_ERR(fmt, ...) CLS_LOG(0, fmt, ##__VA_ARGS__)
-
-void __cls_init();
-
-typedef void *cls_handle_t;
-typedef void *cls_method_handle_t;
 typedef void *cls_filter_handle_t;
-typedef void *cls_method_context_t;
 typedef int (*cls_method_call_t)(cls_method_context_t ctx,
 				 char *indata, int datalen,
 				 char **outdata, int *outdatalen);
@@ -65,8 +35,6 @@ typedef struct {
 } cls_deps_t;
 
 /* class utils */
-extern int cls_log(int level, const char *format, ...)
-  __attribute__((__format__(printf, 2, 3)));
 extern void *cls_alloc(size_t size);
 extern void cls_free(void *p);
 
@@ -85,7 +53,6 @@ extern int cls_get_request_origin(cls_method_context_t hctx,
                                   entity_inst_t *origin);
 
 /* class registration api */
-extern int cls_register(const char *name, cls_handle_t *handle);
 extern int cls_unregister(cls_handle_t);
 
 extern int cls_register_method(cls_handle_t hclass, const char *method, int flags,
@@ -113,9 +80,6 @@ extern void class_fini(void);
 #ifdef __cplusplus
 }
 
-typedef int (*cls_method_cxx_call_t)(cls_method_context_t ctx,
-				     class buffer::list *inbl, class buffer::list *outbl);
-
 class PGLSFilter {
   CephContext* cct;
 protected:
@@ -150,30 +114,18 @@ public:
 typedef PGLSFilter* (*cls_cxx_filter_factory_t)();
 
 
-extern int cls_register_cxx_method(cls_handle_t hclass, const char *method, int flags,
-				   cls_method_cxx_call_t class_call, cls_method_handle_t *handle);
-
 extern int cls_register_cxx_filter(cls_handle_t hclass,
                                    const std::string &filter_name,
 				   cls_cxx_filter_factory_t fn,
                                    cls_filter_handle_t *handle=NULL);
 
-extern int cls_cxx_create(cls_method_context_t hctx, bool exclusive);
-extern int cls_cxx_remove(cls_method_context_t hctx);
-extern int cls_cxx_stat(cls_method_context_t hctx, uint64_t *size, time_t *mtime);
 extern int cls_cxx_stat2(cls_method_context_t hctx, uint64_t *size, ceph::real_time *mtime);
-extern int cls_cxx_read(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
 extern int cls_cxx_read2(cls_method_context_t hctx, int ofs, int len,
                          bufferlist *bl, uint32_t op_flags);
-extern int cls_cxx_write(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
 extern int cls_cxx_write2(cls_method_context_t hctx, int ofs, int len,
                           bufferlist *bl, uint32_t op_flags);
 extern int cls_cxx_write_full(cls_method_context_t hctx, bufferlist *bl);
-extern int cls_cxx_getxattr(cls_method_context_t hctx, const char *name,
-                            bufferlist *outbl);
 extern int cls_cxx_getxattrs(cls_method_context_t hctx, map<string, bufferlist> *attrset);
-extern int cls_cxx_setxattr(cls_method_context_t hctx, const char *name,
-                            bufferlist *inbl);
 extern int cls_cxx_replace(cls_method_context_t hctx, int ofs, int len, bufferlist *bl);
 extern int cls_cxx_snap_revert(cls_method_context_t hctx, snapid_t snapid);
 extern int cls_cxx_map_clear(cls_method_context_t hctx);
@@ -189,10 +141,6 @@ extern int cls_cxx_map_get_vals(cls_method_context_t hctx,
                                 uint64_t max_to_get,
                                 std::map<string, bufferlist> *vals);
 extern int cls_cxx_map_read_header(cls_method_context_t hctx, bufferlist *outbl);
-extern int cls_cxx_map_get_val(cls_method_context_t hctx,
-                               const string &key, bufferlist *outbl);
-extern int cls_cxx_map_set_val(cls_method_context_t hctx,
-                               const string &key, bufferlist *inbl);
 extern int cls_cxx_map_set_vals(cls_method_context_t hctx,
                                 const std::map<string, bufferlist> *map);
 extern int cls_cxx_map_write_header(cls_method_context_t hctx, bufferlist *inbl);

src/test/CMakeLists.txt

@@ -12,6 +12,7 @@ add_subdirectory(cls_hello)
 add_subdirectory(cls_lock)
 add_subdirectory(cls_log)
 add_subdirectory(cls_numops)
+add_subdirectory(cls_sdk)
 if(WITH_RBD)
   add_subdirectory(cls_rbd)
 endif(WITH_RBD)

src/test/cls_sdk/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_executable(ceph_test_cls_sdk
+  test_cls_sdk.cc
+  )
+set_target_properties(ceph_test_cls_sdk PROPERTIES COMPILE_FLAGS
+  ${UNITTEST_CXX_FLAGS})
+target_link_libraries(ceph_test_cls_sdk
+  librados
+  global
+  ${EXTRALIBS}
+  ${BLKID_LIBRARIES}
+  ${CMAKE_DL_LIBS}
+  radostest
+  ${UNITTEST_LIBS}
+  )
+install(TARGETS
+  ceph_test_cls_sdk
+  DESTINATION ${CMAKE_INSTALL_BINDIR})

src/test/cls_sdk/test_cls_sdk.cc

@@ -0,0 +1,35 @@
+#include <iostream>
+#include <errno.h>
+
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+using namespace librados;
+
+TEST(ClsSDK, TestSDKCoverageWrite) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  ASSERT_EQ(0, ioctx.exec("myobject", "sdk", "test_coverage_write", in, out));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+
+TEST(ClsSDK, TestSDKCoverageReplay) {
+  Rados cluster;
+  std::string pool_name = get_temp_pool_name();
+  ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
+  IoCtx ioctx;
+  cluster.ioctx_create(pool_name.c_str(), ioctx);
+
+  bufferlist in, out;
+  ASSERT_EQ(0, ioctx.exec("myobject", "sdk", "test_coverage_write", in, out));
+  ASSERT_EQ(0, ioctx.exec("myobject", "sdk", "test_coverage_replay", in, out));
+
+  ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
+}
+