From ec9b4793b4bf4417e89fad12ef8b8a09eef4052f Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 6 May 2015 12:38:44 +0100 Subject: [PATCH] cls: add CephFS object class For use in cephfs-data-scan for file size and mtime recovery, by doing a set-if-greater for size/mtime xattrs on the 0th object of a file. Signed-off-by: John Spray --- src/cls/CMakeLists.txt | 11 +++ src/cls/Makefile-client.am | 7 +- src/cls/Makefile-server.am | 5 + src/cls/cephfs/cls_cephfs.cc | 143 ++++++++++++++++++++++++++++ src/cls/cephfs/cls_cephfs.h | 127 ++++++++++++++++++++++++ src/cls/cephfs/cls_cephfs_client.cc | 131 +++++++++++++++++++++++++ src/cls/cephfs/cls_cephfs_client.h | 25 +++++ 7 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 src/cls/cephfs/cls_cephfs.cc create mode 100644 src/cls/cephfs/cls_cephfs.h create mode 100644 src/cls/cephfs/cls_cephfs_client.cc create mode 100644 src/cls/cephfs/cls_cephfs_client.h diff --git a/src/cls/CMakeLists.txt b/src/cls/CMakeLists.txt index 2c9c305b2ff..bb050a15f08 100644 --- a/src/cls/CMakeLists.txt +++ b/src/cls/CMakeLists.txt @@ -96,3 +96,14 @@ if (WITH_RADOSGW) rgw/cls_rgw_types.cc rgw/cls_rgw_ops.cc) endif (WITH_RADOSGW) + +# cls_cephfs +if (WITH_CEPHFS) + add_library(cls_cephfs SHARED + rgw/cls_cephfs.cc) + set_target_properties(cls_cephfs PROPERTIES VERSION "1.0.0" SOVERSION "1") + install(TARGETS cls_cephfs DESTINATION lib/rados-classes) + + add_library(cls_cephfs_client + rgw/cls_cephfs_client.cc) +endif (WITH_CEPHFS) diff --git a/src/cls/Makefile-client.am b/src/cls/Makefile-client.am index 70a76daeff8..aa4a4e6054b 100644 --- a/src/cls/Makefile-client.am +++ b/src/cls/Makefile-client.am @@ -48,6 +48,9 @@ DENCODER_DEPS += libcls_user_client.a noinst_LIBRARIES += libcls_user_client.a +libcls_cephfs_client_la_SOURCES = cls/cephfs/cls_cephfs_client.cc +noinst_LTLIBRARIES += libcls_cephfs_client.la + noinst_HEADERS += \ cls/lock/cls_lock_types.h \ cls/lock/cls_lock_ops.h \ @@ -73,4 +76,6 @@ noinst_HEADERS += \ cls/rgw/cls_rgw_types.h \ cls/user/cls_user_client.h \ cls/user/cls_user_ops.h \ - cls/user/cls_user_types.h + cls/user/cls_user_types.h \ + cls/cephfs/cls_cephfs.h \ + cls/cephfs/cls_cephfs_client.h diff --git a/src/cls/Makefile-server.am b/src/cls/Makefile-server.am index ee4cb2b6b0e..7af69ba18db 100644 --- a/src/cls/Makefile-server.am +++ b/src/cls/Makefile-server.am @@ -57,4 +57,9 @@ libcls_rgw_la_SOURCES = \ libcls_rgw_la_LIBADD = libjson_spirit.la $(PTHREAD_LIBS) $(EXTRALIBS) libcls_rgw_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' radoslib_LTLIBRARIES += libcls_rgw.la + +libcls_cephfs_la_SOURCES = cls/cephfs/cls_cephfs.cc +libcls_cephfs_la_LIBADD = $(PTHREAD_LIBS) $(EXTRALIBS) +libcls_cephfs_la_LDFLAGS = ${AM_LDFLAGS} -module -avoid-version -shared -export-symbols-regex '.*__cls_.*' +radoslib_LTLIBRARIES += libcls_cephfs.la endif # WITH_OSD diff --git a/src/cls/cephfs/cls_cephfs.cc b/src/cls/cephfs/cls_cephfs.cc new file mode 100644 index 00000000000..f58f0de39af --- /dev/null +++ b/src/cls/cephfs/cls_cephfs.cc @@ -0,0 +1,143 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include +#include +#include + +#include "objclass/objclass.h" + +#include "cls_cephfs.h" + +CLS_VER(1,0) +CLS_NAME(cephfs_size_scan) + +cls_handle_t h_class; +cls_method_handle_t h_accumulate_inode_metadata; + + + +std::ostream &operator<<(std::ostream &out, ObjCeiling &in) +{ + out << "id: " << in.id << " size: " << in.size; + return out; +} + + +/** + * Set a named xattr to a given value, if and only if the xattr + * is not already set to a greater value. + * + * If the xattr is missing, then it is set to the input integer. + * + * @param xattr_name: name of xattr to compare against and set + * @param input_val: candidate new value, of ::encode()'able type + * @returns 0 on success (irrespective of whether our new value + * was used) else an error code + */ +template +static int set_if_greater(cls_method_context_t hctx, + const std::string &xattr_name, const A input_val) +{ + bufferlist existing_val_bl; + + bool set_val = false; + int r = cls_cxx_getxattr(hctx, xattr_name.c_str(), &existing_val_bl); + if (r == -ENOENT || existing_val_bl.length() == 0) { + set_val = true; + } else if (r >= 0) { + bufferlist::iterator existing_p = existing_val_bl.begin(); + try { + A existing_val; + ::decode(existing_val, existing_p); + if (!existing_p.end()) { + // Trailing junk? Consider it invalid and overwrite + set_val = true; + } else { + // Valid existing value, do comparison + set_val = input_val > existing_val; + } + } catch (const buffer::error &err) { + // Corrupt or empty existing value, overwrite it + set_val = true; + } + } else { + return r; + } + + // Conditionally set the new xattr + if (set_val) { + bufferlist set_bl; + ::encode(input_val, set_bl); + return cls_cxx_setxattr(hctx, xattr_name.c_str(), &set_bl); + } else { + return 0; + } +} + +static int accumulate_inode_metadata(cls_method_context_t hctx, + bufferlist *in, bufferlist *out) +{ + assert(in != NULL); + assert(out != NULL); + + int r = 0; + + // Decode `in` + bufferlist::iterator q = in->begin(); + AccumulateArgs args; + try { + args.decode(q); + } catch (const buffer::error &err) { + return -EINVAL; + } + + ObjCeiling ceiling(args.obj_index, args.obj_size); + r = set_if_greater(hctx, args.obj_xattr_name, ceiling); + if (r < 0) { + return r; + } + + r = set_if_greater(hctx, args.mtime_xattr_name, args.mtime); + if (r < 0) { + return r; + } + + r = set_if_greater(hctx, args.obj_size_xattr_name, args.obj_size); + if (r < 0) { + return r; + } + + return 0; +} + +/** + * initialize class + * + * We do two things here: we register the new class, and then register + * all of the class's methods. + */ +void __cls_init() +{ + // this log message, at level 0, will always appear in the ceph-osd + // log file. + CLS_LOG(0, "loading cephfs_size_scan"); + + cls_register("cephfs", &h_class); + cls_register_cxx_method(h_class, "accumulate_inode_metadata", + CLS_METHOD_WR | CLS_METHOD_RD, + accumulate_inode_metadata, &h_accumulate_inode_metadata); +} + diff --git a/src/cls/cephfs/cls_cephfs.h b/src/cls/cephfs/cls_cephfs.h new file mode 100644 index 00000000000..2cbc76b47ab --- /dev/null +++ b/src/cls/cephfs/cls_cephfs.h @@ -0,0 +1,127 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include "include/encoding.h" + +/** + * Value class for the xattr we'll use to accumulate + * the highest object seen for a given inode + */ +class ObjCeiling { + public: + uint64_t id; + uint64_t size; + + ObjCeiling() + : id(0), size(0) + {} + + ObjCeiling(uint64_t id_, uint64_t size_) + : id(id_), size(size_) + {} + + bool operator >(ObjCeiling const &rhs) const + { + return id > rhs.id; + } + + void encode(bufferlist &bl) const + { + ENCODE_START(1, 1, bl); + ::encode(id, bl); + ::encode(size, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator &p) + { + DECODE_START(1, p); + ::decode(id, p); + ::decode(size, p); + DECODE_FINISH(p); + } +}; +WRITE_CLASS_ENCODER(ObjCeiling) + +class AccumulateArgs +{ +public: + uint64_t obj_index; + uint64_t obj_size; + time_t mtime; + std::string obj_xattr_name; + std::string mtime_xattr_name; + std::string obj_size_xattr_name; + + AccumulateArgs( + uint64_t obj_index_, + uint64_t obj_size_, + time_t mtime_, + std::string obj_xattr_name_, + std::string mtime_xattr_name_, + std::string obj_size_xattr_name_) + : obj_index(obj_index_), + obj_size(obj_size_), + mtime(mtime_), + obj_xattr_name(obj_xattr_name_), + mtime_xattr_name(mtime_xattr_name_), + obj_size_xattr_name(obj_size_xattr_name_) + {} + + AccumulateArgs() + : obj_index(0), obj_size(0), mtime(0) + {} + + void encode(bufferlist &bl) const + { + ENCODE_START(1, 1, bl); + ::encode(obj_xattr_name, bl); + ::encode(mtime_xattr_name, bl); + ::encode(obj_size_xattr_name, bl); + ::encode(obj_index, bl); + ::encode(obj_size, bl); + ::encode(mtime, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator &bl) + { + DECODE_START(1, bl); + ::decode(obj_xattr_name, bl); + ::decode(mtime_xattr_name, bl); + ::decode(obj_size_xattr_name, bl); + ::decode(obj_index, bl); + ::decode(obj_size, bl); + ::decode(mtime, bl); + DECODE_FINISH(bl); + } +}; + +class AccumulateResult +{ +public: + // Index of the highest-indexed object seen + uint64_t ceiling_obj_index; + // Size of the highest-index object seen + uint64_t ceiling_obj_size; + // Largest object seen + uint64_t max_obj_size; + // Highest mtime seen + time_t max_mtime; + + AccumulateResult() + : ceiling_obj_index(0), ceiling_obj_size(0), max_obj_size(0), max_mtime(0) + {} +}; + diff --git a/src/cls/cephfs/cls_cephfs_client.cc b/src/cls/cephfs/cls_cephfs_client.cc new file mode 100644 index 00000000000..d135922699e --- /dev/null +++ b/src/cls/cephfs/cls_cephfs_client.cc @@ -0,0 +1,131 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2015 Red Hat + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#include "cls_cephfs_client.h" + +#include "mds/CInode.h" + +#define XATTR_CEILING "scan_ceiling" +#define XATTR_MAX_MTIME "scan_max_mtime" +#define XATTR_MAX_SIZE "scan_max_size" + +int ClsCephFSClient::accumulate_inode_metadata( + librados::IoCtx &ctx, + inodeno_t inode_no, + const uint64_t obj_index, + const uint64_t obj_size, + const time_t mtime) +{ + AccumulateArgs args( + obj_index, + obj_size, + mtime, + XATTR_CEILING, + XATTR_MAX_MTIME, + XATTR_MAX_SIZE); + + // Generate 0th object name, where we will accumulate sizes/mtimes + object_t zeroth_object = InodeStore::get_object_name(inode_no, frag_t(), ""); + + // Construct a librados operation invoking our class method + librados::ObjectReadOperation op; + bufferlist inbl; + args.encode(inbl); + op.exec("cephfs", "accumulate_inode_metadata", inbl); + + // Execute op + bufferlist outbl; + return ctx.operate(zeroth_object.name, &op, &outbl); +} + +int ClsCephFSClient::fetch_inode_accumulate_result( + librados::IoCtx &ctx, + const std::string &oid, + inode_backtrace_t *backtrace, + AccumulateResult *result) +{ + assert(backtrace != NULL); + assert(result != NULL); + + librados::ObjectReadOperation op; + + int scan_ceiling_r = 0; + bufferlist scan_ceiling_bl; + op.getxattr(XATTR_CEILING, &scan_ceiling_bl, &scan_ceiling_r); + + int scan_max_size_r = 0; + bufferlist scan_max_size_bl; + op.getxattr(XATTR_MAX_SIZE, &scan_max_size_bl, &scan_max_size_r); + + int scan_max_mtime_r = 0; + bufferlist scan_max_mtime_bl; + op.getxattr(XATTR_MAX_MTIME, &scan_max_mtime_bl, &scan_max_mtime_r); + + int parent_r = 0; + bufferlist parent_bl; + op.getxattr("parent", &parent_bl, &parent_r); + + bufferlist op_bl; + int r = ctx.operate(oid, &op, &op_bl); + if (r < 0 && r != -ENODATA) { + // ENODATA acceptable from parent getxattr (just means there happens + // not to be a backtrace) + return r; + } + + // Load scan_ceiling + try { + bufferlist::iterator scan_ceiling_bl_iter = scan_ceiling_bl.begin(); + ObjCeiling ceiling; + ceiling.decode(scan_ceiling_bl_iter); + result->ceiling_obj_index = ceiling.id; + result->ceiling_obj_size = ceiling.size; + } catch (const buffer::error &err) { + //dout(4) << "Invalid size attr on '" << oid << "'" << dendl; + return -EINVAL; + } + + // Load scan_max_size + try { + bufferlist::iterator scan_max_size_bl_iter = scan_max_size_bl.begin(); + ::decode(result->max_obj_size, scan_max_size_bl_iter); + } catch (const buffer::error &err) { + //dout(4) << "Invalid size attr on '" << oid << "'" << dendl; + return -EINVAL; + } + + // Load scan_max_mtime + try { + bufferlist::iterator scan_max_mtime_bl_iter = scan_max_mtime_bl.begin(); + ::decode(result->max_mtime, scan_max_mtime_bl_iter); + } catch (const buffer::error &err) { + //dout(4) << "Invalid size attr on '" << oid << "'" << dendl; + return -EINVAL; + } + + // Deserialize backtrace + if (parent_bl.length()) { + try { + bufferlist::iterator q = parent_bl.begin(); + backtrace->decode(q); + } catch (buffer::error &e) { + //dout(4) << "Corrupt backtrace on '" << oid << "': " << e << dendl; + return -EINVAL; + } + } + + return 0; +} + diff --git a/src/cls/cephfs/cls_cephfs_client.h b/src/cls/cephfs/cls_cephfs_client.h new file mode 100644 index 00000000000..5448a31505b --- /dev/null +++ b/src/cls/cephfs/cls_cephfs_client.h @@ -0,0 +1,25 @@ + +#include "include/rados/librados.hpp" +#include "mds/mdstypes.h" + +#include "cls_cephfs.h" + +class AccumulateArgs; + +class ClsCephFSClient +{ + public: + static int accumulate_inode_metadata( + librados::IoCtx &ctx, + inodeno_t inode_no, + const uint64_t obj_index, + const uint64_t obj_size, + const time_t mtime); + + static int fetch_inode_accumulate_result( + librados::IoCtx &ctx, + const std::string &oid, + inode_backtrace_t *backtrace, + AccumulateResult *result); +}; +