cls: add CephFS object class

For use in cephfs-data-scan for file size and mtime
recovery, by doing a set-if-greater for size/mtime
xattrs on the 0th object of a file.

Signed-off-by: John Spray <john.spray@redhat.com>

src/cls/CMakeLists.txt

@@ -96,3 +96,14 @@ if (WITH_RADOSGW)
     rgw/cls_rgw_types.cc
     rgw/cls_rgw_ops.cc)
 endif (WITH_RADOSGW)
+
+# cls_cephfs
+if (WITH_CEPHFS)
+  add_library(cls_cephfs SHARED
+    rgw/cls_cephfs.cc)
+  set_target_properties(cls_cephfs PROPERTIES VERSION "1.0.0" SOVERSION "1")
+  install(TARGETS cls_cephfs DESTINATION lib/rados-classes)
+
+  add_library(cls_cephfs_client
+    rgw/cls_cephfs_client.cc)
+endif (WITH_CEPHFS)

src/cls/Makefile-client.am

@@ -48,6 +48,9 @@ DENCODER_DEPS += libcls_user_client.a
 
 noinst_LIBRARIES += libcls_user_client.a
 
+libcls_cephfs_client_la_SOURCES = cls/cephfs/cls_cephfs_client.cc
+noinst_LTLIBRARIES += libcls_cephfs_client.la
+
 noinst_HEADERS += \
 	cls/lock/cls_lock_types.h \
 	cls/lock/cls_lock_ops.h \
@@ -73,4 +76,6 @@ noinst_HEADERS += \
 	cls/rgw/cls_rgw_types.h \
 	cls/user/cls_user_client.h \
 	cls/user/cls_user_ops.h \
-	cls/user/cls_user_types.h
+	cls/user/cls_user_types.h \
+	cls/cephfs/cls_cephfs.h \
+	cls/cephfs/cls_cephfs_client.h

src/cls/Makefile-server.am

@@ -57,4 +57,9 @@ libcls_rgw_la_SOURCES = \
 libcls_rgw_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS)
 libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
 radoslib_LTLIBRARIES += libcls_rgw.la
+
+libcls_cephfs_la_SOURCES = cls/cephfs/cls_cephfs.cc
+libcls_cephfs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS)
+libcls_cephfs_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*'
+radoslib_LTLIBRARIES += libcls_cephfs.la
 endif # WITH_OSD

src/cls/cephfs/cls_cephfs.cc

@@ -0,0 +1,143 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#include <string>
+#include <errno.h>
+#include <sstream>
+
+#include "objclass/objclass.h"
+
+#include "cls_cephfs.h"
+
+CLS_VER(1,0)
+CLS_NAME(cephfs_size_scan)
+
+cls_handle_t h_class;
+cls_method_handle_t h_accumulate_inode_metadata;
+
+
+
+std::ostream &operator<<(std::ostream &out, ObjCeiling &in)
+{
+  out << "id: " << in.id << " size: " << in.size;
+  return out;
+}
+
+
+/**
+ * Set a named xattr to a given value, if and only if the xattr
+ * is not already set to a greater value.
+ *
+ * If the xattr is missing, then it is set to the input integer.
+ *
+ * @param xattr_name: name of xattr to compare against and set
+ * @param input_val: candidate new value, of ::encode()'able type
+ * @returns 0 on success (irrespective of whether our new value
+ *          was used) else an error code
+ */
+template <typename A>
+static int set_if_greater(cls_method_context_t hctx,
+    const std::string &xattr_name, const A input_val)
+{
+  bufferlist existing_val_bl;
+
+  bool set_val = false;
+  int r = cls_cxx_getxattr(hctx, xattr_name.c_str(), &existing_val_bl);
+  if (r == -ENOENT || existing_val_bl.length() == 0) {
+    set_val = true;
+  } else if (r >= 0) {
+    bufferlist::iterator existing_p = existing_val_bl.begin();
+    try {
+      A existing_val;
+      ::decode(existing_val, existing_p);
+      if (!existing_p.end()) {
+        // Trailing junk?  Consider it invalid and overwrite
+        set_val = true;
+      } else {
+        // Valid existing value, do comparison
+        set_val = input_val > existing_val;
+      }
+    } catch (const buffer::error &err) {
+      // Corrupt or empty existing value, overwrite it
+      set_val = true;
+    }
+  } else {
+    return r;
+  }
+
+  // Conditionally set the new xattr
+  if (set_val) {
+    bufferlist set_bl;
+    ::encode(input_val, set_bl);
+    return cls_cxx_setxattr(hctx, xattr_name.c_str(), &set_bl);
+  } else {
+    return 0;
+  }
+}
+
+static int accumulate_inode_metadata(cls_method_context_t hctx,
+    bufferlist *in, bufferlist *out)
+{
+  assert(in != NULL);
+  assert(out != NULL);
+
+  int r = 0;
+
+  // Decode `in`
+  bufferlist::iterator q = in->begin();
+  AccumulateArgs args;
+  try {
+    args.decode(q);
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  ObjCeiling ceiling(args.obj_index, args.obj_size);
+  r = set_if_greater(hctx, args.obj_xattr_name, ceiling);
+  if (r < 0) {
+    return r;
+  }
+
+  r = set_if_greater(hctx, args.mtime_xattr_name, args.mtime);
+  if (r < 0) {
+    return r;
+  }
+
+  r = set_if_greater(hctx, args.obj_size_xattr_name, args.obj_size);
+  if (r < 0) {
+    return r;
+  }
+
+  return 0;
+}
+
+/**
+ * initialize class
+ *
+ * We do two things here: we register the new class, and then register
+ * all of the class's methods.
+ */
+void __cls_init()
+{
+  // this log message, at level 0, will always appear in the ceph-osd
+  // log file.
+  CLS_LOG(0, "loading cephfs_size_scan");
+
+  cls_register("cephfs", &h_class);
+  cls_register_cxx_method(h_class, "accumulate_inode_metadata",
+			  CLS_METHOD_WR | CLS_METHOD_RD,
+			  accumulate_inode_metadata, &h_accumulate_inode_metadata);
+}
+

src/cls/cephfs/cls_cephfs.h

@@ -0,0 +1,127 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "include/encoding.h"
+
+/**
+ * Value class for the xattr we'll use to accumulate
+ * the highest object seen for a given inode
+ */
+class ObjCeiling {
+  public:
+    uint64_t id;
+    uint64_t size;
+
+    ObjCeiling()
+      : id(0), size(0)
+    {}
+
+    ObjCeiling(uint64_t id_, uint64_t size_)
+      : id(id_), size(size_)
+    {}
+
+    bool operator >(ObjCeiling const &rhs) const
+    {
+      return id > rhs.id;
+    }
+
+    void encode(bufferlist &bl) const
+    {
+      ENCODE_START(1, 1, bl);
+      ::encode(id, bl);
+      ::encode(size, bl);
+      ENCODE_FINISH(bl);
+    }
+
+    void decode(bufferlist::iterator &p)
+    {
+      DECODE_START(1, p);
+      ::decode(id, p);
+      ::decode(size, p);
+      DECODE_FINISH(p);
+    }
+};
+WRITE_CLASS_ENCODER(ObjCeiling)
+
+class AccumulateArgs
+{
+public:
+  uint64_t obj_index;
+  uint64_t obj_size;
+  time_t mtime;
+  std::string obj_xattr_name;
+  std::string mtime_xattr_name;
+  std::string obj_size_xattr_name;
+
+  AccumulateArgs(
+      uint64_t obj_index_,
+      uint64_t obj_size_,
+      time_t mtime_,
+      std::string obj_xattr_name_,
+      std::string mtime_xattr_name_,
+      std::string obj_size_xattr_name_)
+   : obj_index(obj_index_),
+     obj_size(obj_size_),
+     mtime(mtime_),
+     obj_xattr_name(obj_xattr_name_),
+     mtime_xattr_name(mtime_xattr_name_),
+     obj_size_xattr_name(obj_size_xattr_name_)
+  {}
+
+  AccumulateArgs()
+    : obj_index(0), obj_size(0), mtime(0)
+  {}
+
+  void encode(bufferlist &bl) const
+  {
+    ENCODE_START(1, 1, bl);
+    ::encode(obj_xattr_name, bl);
+    ::encode(mtime_xattr_name, bl);
+    ::encode(obj_size_xattr_name, bl);
+    ::encode(obj_index, bl);
+    ::encode(obj_size, bl);
+    ::encode(mtime, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator &bl)
+  {
+    DECODE_START(1, bl);
+    ::decode(obj_xattr_name, bl);
+    ::decode(mtime_xattr_name, bl);
+    ::decode(obj_size_xattr_name, bl);
+    ::decode(obj_index, bl);
+    ::decode(obj_size, bl);
+    ::decode(mtime, bl);
+    DECODE_FINISH(bl);
+  }
+};
+
+class AccumulateResult
+{
+public:
+  // Index of the highest-indexed object seen
+  uint64_t ceiling_obj_index;
+  // Size of the highest-index object seen
+  uint64_t ceiling_obj_size;
+  // Largest object seen
+  uint64_t max_obj_size;
+  // Highest mtime seen
+  time_t   max_mtime;
+
+  AccumulateResult()
+    : ceiling_obj_index(0), ceiling_obj_size(0), max_obj_size(0), max_mtime(0)
+  {}
+};
+

src/cls/cephfs/cls_cephfs_client.cc

@@ -0,0 +1,131 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+
+#include "cls_cephfs_client.h"
+
+#include "mds/CInode.h"
+
+#define XATTR_CEILING "scan_ceiling"
+#define XATTR_MAX_MTIME "scan_max_mtime"
+#define XATTR_MAX_SIZE "scan_max_size"
+
+int ClsCephFSClient::accumulate_inode_metadata(
+  librados::IoCtx &ctx,
+  inodeno_t inode_no,
+  const uint64_t obj_index,
+  const uint64_t obj_size,
+  const time_t mtime)
+{
+  AccumulateArgs args(
+      obj_index,
+      obj_size,
+      mtime,
+      XATTR_CEILING,
+      XATTR_MAX_MTIME,
+      XATTR_MAX_SIZE);
+
+  // Generate 0th object name, where we will accumulate sizes/mtimes
+  object_t zeroth_object = InodeStore::get_object_name(inode_no, frag_t(), "");
+
+  // Construct a librados operation invoking our class method
+  librados::ObjectReadOperation op;
+  bufferlist inbl;
+  args.encode(inbl);
+  op.exec("cephfs", "accumulate_inode_metadata", inbl);
+
+  // Execute op
+  bufferlist outbl;
+  return ctx.operate(zeroth_object.name, &op, &outbl);
+}
+
+int ClsCephFSClient::fetch_inode_accumulate_result(
+  librados::IoCtx &ctx,
+  const std::string &oid,
+  inode_backtrace_t *backtrace,
+  AccumulateResult *result)
+{
+  assert(backtrace != NULL);
+  assert(result != NULL);
+
+  librados::ObjectReadOperation op;
+
+  int scan_ceiling_r = 0;
+  bufferlist scan_ceiling_bl;
+  op.getxattr(XATTR_CEILING, &scan_ceiling_bl, &scan_ceiling_r);
+
+  int scan_max_size_r = 0;
+  bufferlist scan_max_size_bl;
+  op.getxattr(XATTR_MAX_SIZE, &scan_max_size_bl, &scan_max_size_r);
+
+  int scan_max_mtime_r = 0;
+  bufferlist scan_max_mtime_bl;
+  op.getxattr(XATTR_MAX_MTIME, &scan_max_mtime_bl, &scan_max_mtime_r);
+
+  int parent_r = 0;
+  bufferlist parent_bl;
+  op.getxattr("parent", &parent_bl, &parent_r);
+
+  bufferlist op_bl;
+  int r = ctx.operate(oid, &op, &op_bl);
+  if (r < 0 && r != -ENODATA) {
+    // ENODATA acceptable from parent getxattr (just means there happens
+    // not to be a backtrace)
+    return r;
+  }
+
+  // Load scan_ceiling
+  try {
+    bufferlist::iterator scan_ceiling_bl_iter = scan_ceiling_bl.begin();
+    ObjCeiling ceiling;
+    ceiling.decode(scan_ceiling_bl_iter);
+    result->ceiling_obj_index = ceiling.id;
+    result->ceiling_obj_size = ceiling.size;
+  } catch (const buffer::error &err) {
+    //dout(4) << "Invalid size attr on '" << oid << "'" << dendl;
+    return -EINVAL;
+  }
+
+  // Load scan_max_size
+  try {
+    bufferlist::iterator scan_max_size_bl_iter = scan_max_size_bl.begin();
+    ::decode(result->max_obj_size, scan_max_size_bl_iter);
+  } catch (const buffer::error &err) {
+    //dout(4) << "Invalid size attr on '" << oid << "'" << dendl;
+    return -EINVAL;
+  }
+
+  // Load scan_max_mtime
+  try {
+    bufferlist::iterator scan_max_mtime_bl_iter = scan_max_mtime_bl.begin();
+    ::decode(result->max_mtime, scan_max_mtime_bl_iter);
+  } catch (const buffer::error &err) {
+    //dout(4) << "Invalid size attr on '" << oid << "'" << dendl;
+    return -EINVAL;
+  }
+
+  // Deserialize backtrace
+  if (parent_bl.length()) {
+    try {
+      bufferlist::iterator q = parent_bl.begin();
+      backtrace->decode(q);
+    } catch (buffer::error &e) {
+      //dout(4) << "Corrupt backtrace on '" << oid << "': " << e << dendl;
+      return -EINVAL;
+    }
+  }
+
+  return 0;
+}
+

src/cls/cephfs/cls_cephfs_client.h

@@ -0,0 +1,25 @@
+
+#include "include/rados/librados.hpp"
+#include "mds/mdstypes.h"
+
+#include "cls_cephfs.h"
+
+class AccumulateArgs;
+
+class ClsCephFSClient
+{
+  public:
+  static int accumulate_inode_metadata(
+      librados::IoCtx &ctx,
+      inodeno_t inode_no,
+      const uint64_t obj_index,
+      const uint64_t obj_size,
+      const time_t mtime);
+
+  static int fetch_inode_accumulate_result(
+      librados::IoCtx &ctx,
+      const std::string &oid,
+      inode_backtrace_t *backtrace,
+      AccumulateResult *result);
+};
+