RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-23 11:07:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23396 from linuxbox2/wip-rgw-xattrs-2

Browse Source 
rgw_file:  expose RGW user-defined attributes
This commit is contained in:
Matt Benjamin 2020-12-17 09:30:55 -05:00 committed by GitHub
commit 05d75ae32e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1112 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src/include/rados/rgw_file.h
View File

@ -26,8 +26,8 @@ extern "C" {
#endif
#define LIBRGW_FILE_VER_MAJOR 1
#define LIBRGW_FILE_VER_MINOR 1
#define LIBRGW_FILE_VER_EXTRA 7
#define LIBRGW_FILE_VER_MINOR 2
#define LIBRGW_FILE_VER_EXTRA 0
#define LIBRGW_FILE_VERSION(maj, min, extra) ((maj << 16) + (min << 8) + extra)
#define LIBRGW_FILE_VERSION_CODE LIBRGW_FILE_VERSION(LIBRGW_FILE_VER_MAJOR, LIBRGW_FILE_VER_MINOR, LIBRGW_FILE_VER_EXTRA)
@ -377,6 +377,53 @@ int rgw_fsync(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
int rgw_commit(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
uint64_t offset, uint64_t length, uint32_t flags);
/*
extended attributes
*/
typedef struct rgw_xattrstr
{
char *val;
uint32_t len;
} rgw_xattrstr;
typedef struct rgw_xattr
{
rgw_xattrstr key;
rgw_xattrstr val;
} rgw_xattr;
typedef struct rgw_xattrlist
{
rgw_xattr *xattrs;
uint32_t xattr_cnt;
} rgw_xattrlist;
#define RGW_GETXATTR_FLAG_NONE 0x0000
typedef int (*rgw_getxattr_cb)(rgw_xattrlist *attrs, void *arg,
uint32_t flags);
int rgw_getxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrlist *attrs, rgw_getxattr_cb cb, void *cb_arg,
uint32_t flags);
#define RGW_LSXATTR_FLAG_NONE 0x0000
#define RGW_LSXATTR_FLAG_STOP 0x0001
int rgw_lsxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrstr *filter_prefix /* unimplemented for now */,
rgw_getxattr_cb cb, void *cb_arg, uint32_t flags);
#define RGW_SETXATTR_FLAG_NONE 0x0000
int rgw_setxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrlist *attrs, uint32_t flags);
#define RGW_RMXATTR_FLAG_NONE 0x0000
int rgw_rmxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrlist *attrs, uint32_t flags);
#ifdef __cplusplus
}
#endif

306
src/rgw/rgw_file.cc
View File

@ -69,6 +69,33 @@ namespace rgw {
return 0;
}
class XattrHash
{
public:
std::size_t operator()(const rgw_xattrstr& att) const noexcept {
return XXH64(att.val, att.len, 5882300);
}
};
class XattrEqual
{
public:
bool operator()(const rgw_xattrstr& lhs, const rgw_xattrstr& rhs) const {
return ((lhs.len == rhs.len) &&
(strncmp(lhs.val, rhs.val, lhs.len) == 0));
}
};
/* well-known attributes */
static const std::unordered_set<
rgw_xattrstr, XattrHash, XattrEqual> rgw_exposed_attrs = {
rgw_xattrstr{const_cast<char*>(RGW_ATTR_ETAG), sizeof(RGW_ATTR_ETAG)-1}
};
static inline bool is_exposed_attr(const rgw_xattrstr& k) {
return (rgw_exposed_attrs.find(k) != rgw_exposed_attrs.end());
}
LookupFHResult RGWLibFS::stat_bucket(RGWFileHandle* parent, const char *path,
RGWLibFS::BucketStats& bs,
uint32_t flags)
@ -931,7 +958,242 @@ namespace rgw {
return 0;
} /* RGWLibFS::setattr */
/* called under rgw_fh->mtx held */
static inline std::string prefix_xattr_keystr(const rgw_xattrstr& key) {
std::string keystr;
keystr.reserve(sizeof(RGW_ATTR_META_PREFIX) + key.len);
keystr += {RGW_ATTR_META_PREFIX};
keystr += string{key.val, key.len};
return keystr;
}
static inline std::string_view unprefix_xattr_keystr(const std::string& key)
{
std::string_view svk{key};
auto pos = svk.find(RGW_ATTR_META_PREFIX);
if (pos == std::string_view::npos) {
return std::string_view{""};
} else if (pos == 0) {
svk.remove_prefix(sizeof(RGW_ATTR_META_PREFIX)-1);
}
return svk;
}
int RGWLibFS::getxattrs(RGWFileHandle* rgw_fh, rgw_xattrlist *attrs,
rgw_getxattr_cb cb, void *cb_arg,
uint32_t flags)
{
/* cannot store on fs_root, should not on buckets? */
if ((rgw_fh->is_bucket()) ||
(rgw_fh->is_root())) {
return -EINVAL;
}
int rc, rc2, rc3;
string obj_name{rgw_fh->relative_object_name2()};
RGWGetAttrsRequest req(cct, rgwlib.get_store()->get_user(user.user_id),
rgw_fh->bucket_name(), obj_name);
for (uint32_t ix = 0; ix < attrs->xattr_cnt; ++ix) {
auto& xattr = attrs->xattrs[ix];
/* pass exposed attr keys as given, else prefix */
std::string k = is_exposed_attr(xattr.key)
? std::string{xattr.key.val, xattr.key.len}
: prefix_xattr_keystr(xattr.key);
req.emplace_key(std::move(k));
}
if (ldlog_p1(get_context(), ceph_subsys_rgw, 15)) {
lsubdout(get_context(), rgw, 15)
<< __func__
<< " get keys for: "
<< rgw_fh->object_name()
<< " keys:"
<< dendl;
for (const auto& attr: req.get_attrs()) {
lsubdout(get_context(), rgw, 15)
<< "\tkey: " << attr.first << dendl;
}
}
rc = rgwlib.get_fe()->execute_req(&req);
rc2 = req.get_ret();
rc3 = ((rc == 0) && (rc2 == 0)) ? 0 : -EIO;
/* call back w/xattr data */
if (rc3 == 0) {
const auto& attrs = req.get_attrs();
for (const auto& attr : attrs) {
if (!attr.second.has_value())
continue;
const auto& k = attr.first;
const auto& v = attr.second.value();
/* return exposed attr keys as given, else unprefix --
* yes, we could have memoized the exposed check, but
* to be efficient it would need to be saved with
* RGWGetAttrs::attrs, I think */
std::string_view svk =
is_exposed_attr(rgw_xattrstr{const_cast<char*>(k.c_str()),
uint32_t(k.length())})
? k
: unprefix_xattr_keystr(k);
/* skip entries not matching prefix */
if (svk.empty())
continue;
rgw_xattrstr xattr_k = { const_cast<char*>(svk.data()),
uint32_t(svk.length())};
rgw_xattrstr xattr_v =
{const_cast<char*>(const_cast<buffer::list&>(v).c_str()),
uint32_t(v.length())};
rgw_xattr xattr = { xattr_k, xattr_v };
rgw_xattrlist xattrlist = { &xattr, 1 };
cb(&xattrlist, cb_arg, RGW_GETXATTR_FLAG_NONE);
}
}
return rc3;
} /* RGWLibFS::getxattrs */
int RGWLibFS::lsxattrs(
RGWFileHandle* rgw_fh, rgw_xattrstr *filter_prefix, rgw_getxattr_cb cb,
void *cb_arg, uint32_t flags)
{
/* cannot store on fs_root, should not on buckets? */
if ((rgw_fh->is_bucket()) ||
(rgw_fh->is_root())) {
return -EINVAL;
}
int rc, rc2, rc3;
string obj_name{rgw_fh->relative_object_name2()};
RGWGetAttrsRequest req(cct, rgwlib.get_store()->get_user(user.user_id),
rgw_fh->bucket_name(), obj_name);
rc = rgwlib.get_fe()->execute_req(&req);
rc2 = req.get_ret();
rc3 = ((rc == 0) && (rc2 == 0)) ? 0 : -EIO;
/* call back w/xattr data--check for eof */
if (rc3 == 0) {
const auto& keys = req.get_attrs();
for (const auto& k : keys) {
/* return exposed attr keys as given, else unprefix */
std::string_view svk =
is_exposed_attr(rgw_xattrstr{const_cast<char*>(k.first.c_str()),
uint32_t(k.first.length())})
? k.first
: unprefix_xattr_keystr(k.first);
/* skip entries not matching prefix */
if (svk.empty())
continue;
rgw_xattrstr xattr_k = { const_cast<char*>(svk.data()),
uint32_t(svk.length())};
rgw_xattrstr xattr_v = { nullptr, 0 };
rgw_xattr xattr = { xattr_k, xattr_v };
rgw_xattrlist xattrlist = { &xattr, 1 };
auto cbr = cb(&xattrlist, cb_arg, RGW_LSXATTR_FLAG_NONE);
if (cbr & RGW_LSXATTR_FLAG_STOP)
break;
}
}
return rc3;
} /* RGWLibFS::lsxattrs */
int RGWLibFS::setxattrs(RGWFileHandle* rgw_fh, rgw_xattrlist *attrs,
uint32_t flags)
{
/* cannot store on fs_root, should not on buckets? */
if ((rgw_fh->is_bucket()) ||
(rgw_fh->is_root())) {
return -EINVAL;
}
int rc, rc2;
string obj_name{rgw_fh->relative_object_name2()};
RGWSetAttrsRequest req(cct, rgwlib.get_store()->get_user(user.user_id),
rgw_fh->bucket_name(), obj_name);
for (uint32_t ix = 0; ix < attrs->xattr_cnt; ++ix) {
auto& xattr = attrs->xattrs[ix];
buffer::list attr_bl;
/* don't allow storing at RGW_ATTR_META_PREFIX */
if (! (xattr.key.len > 0))
continue;
/* reject lexical match with any exposed attr */
if (is_exposed_attr(xattr.key))
continue;
string k = prefix_xattr_keystr(xattr.key);
attr_bl.append(xattr.val.val, xattr.val.len);
req.emplace_attr(k.c_str(), std::move(attr_bl));
}
/* don't send null requests */
if (! (req.get_attrs().size() > 0)) {
return -EINVAL;
}
rc = rgwlib.get_fe()->execute_req(&req);
rc2 = req.get_ret();
return (((rc == 0) && (rc2 == 0)) ? 0 : -EIO);
} /* RGWLibFS::setxattrs */
int RGWLibFS::rmxattrs(RGWFileHandle* rgw_fh, rgw_xattrlist* attrs,
uint32_t flags)
{
/* cannot store on fs_root, should not on buckets? */
if ((rgw_fh->is_bucket()) ||
(rgw_fh->is_root())) {
return -EINVAL;
}
int rc, rc2;
string obj_name{rgw_fh->relative_object_name2()};
RGWRMAttrsRequest req(cct, rgwlib.get_store()->get_user(user.user_id),
rgw_fh->bucket_name(), obj_name);
for (uint32_t ix = 0; ix < attrs->xattr_cnt; ++ix) {
auto& xattr = attrs->xattrs[ix];
/* don't allow storing at RGW_ATTR_META_PREFIX */
if (! (xattr.key.len > 0)) {
continue;
}
string k = prefix_xattr_keystr(xattr.key);
req.emplace_key(std::move(k));
}
/* don't send null requests */
if (! (req.get_attrs().size() > 0)) {
return -EINVAL;
}
rc = rgwlib.get_fe()->execute_req(&req);
rc2 = req.get_ret();
return (((rc == 0) && (rc2 == 0)) ? 0 : -EIO);
} /* RGWLibFS::rmxattrs */
/* called with rgw_fh->mtx held */
void RGWLibFS::update_fh(RGWFileHandle *rgw_fh)
{
int rc, rc2;
@ -2447,4 +2709,46 @@ int rgw_commit(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
return rgw_fh->commit(offset, length, RGWFileHandle::FLAG_NONE);
}
/*
extended attributes
*/
int rgw_getxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrlist *attrs, rgw_getxattr_cb cb, void *cb_arg,
uint32_t flags)
{
RGWLibFS *fs = static_cast<RGWLibFS*>(rgw_fs->fs_private);
RGWFileHandle* rgw_fh = get_rgwfh(fh);
return fs->getxattrs(rgw_fh, attrs, cb, cb_arg, flags);
}
int rgw_lsxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrstr *filter_prefix /* ignored */,
rgw_getxattr_cb cb, void *cb_arg, uint32_t flags)
{
RGWLibFS *fs = static_cast<RGWLibFS*>(rgw_fs->fs_private);
RGWFileHandle* rgw_fh = get_rgwfh(fh);
return fs->lsxattrs(rgw_fh, filter_prefix, cb, cb_arg, flags);
}
int rgw_setxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrlist *attrs, uint32_t flags)
{
RGWLibFS *fs = static_cast<RGWLibFS*>(rgw_fs->fs_private);
RGWFileHandle* rgw_fh = get_rgwfh(fh);
return fs->setxattrs(rgw_fh, attrs, flags);
}
int rgw_rmxattrs(struct rgw_fs *rgw_fs, struct rgw_file_handle *fh,
rgw_xattrlist *attrs, uint32_t flags)
{
RGWLibFS *fs = static_cast<RGWLibFS*>(rgw_fs->fs_private);
RGWFileHandle* rgw_fh = get_rgwfh(fh);
return fs->rmxattrs(rgw_fh, attrs, flags);
}
} /* extern "C" */

134
src/rgw/rgw_file.h
View File

@ -525,6 +525,14 @@ namespace rgw {
return full_object_name(true /* omit_bucket */);
}
inline std::string relative_object_name2() {
std::string rname = full_object_name(true /* omit_bucket */);
if (is_dir()) {
rname += "/";
}
return rname;
}
inline std::string format_child_name(const std::string& cbasename,
bool is_dir) const {
std::string child_name{relative_object_name()};
@ -1179,6 +1187,16 @@ namespace rgw {
int setattr(RGWFileHandle* rgw_fh, struct stat* st, uint32_t mask,
uint32_t flags);
int getxattrs(RGWFileHandle* rgw_fh, rgw_xattrlist* attrs,
rgw_getxattr_cb cb, void *cb_arg, uint32_t flags);
int lsxattrs(RGWFileHandle* rgw_fh, rgw_xattrstr *filter_prefix,
rgw_getxattr_cb cb, void *cb_arg, uint32_t flags);
int setxattrs(RGWFileHandle* rgw_fh, rgw_xattrlist* attrs, uint32_t flags);
int rmxattrs(RGWFileHandle* rgw_fh, rgw_xattrlist* attrs, uint32_t flags);
void update_fh(RGWFileHandle *rgw_fh);
LookupFHResult stat_bucket(RGWFileHandle* parent, const char *path,
@ -2663,6 +2681,62 @@ public:
}; /* RGWCopyObjRequest */
class RGWGetAttrsRequest : public RGWLibRequest,
public RGWGetAttrs /* RGWOp */
{
public:
const std::string& bucket_name;
const std::string& obj_name;
RGWGetAttrsRequest(CephContext* _cct,
std::unique_ptr<rgw::sal::RGWUser> _user,
const std::string& _bname, const std::string& _oname)
: RGWLibRequest(_cct, std::move(_user)), RGWGetAttrs(),
bucket_name(_bname), obj_name(_oname) {
op = this;
}
const flat_map<std::string, std::optional<buffer::list>>& get_attrs() {
return attrs;
}
virtual bool only_bucket() { return false; }
virtual int op_init() {
// assign store, s, and dialect_handler
RGWObjectCtx* rados_ctx
= static_cast<RGWObjectCtx*>(get_state()->obj_ctx);
// framework promises to call op_init after parent init
assert(rados_ctx);
RGWOp::init(rados_ctx->get_store(), get_state(), this);
op = this; // assign self as op: REQUIRED
return 0;
}
virtual int header_init() {
struct req_state* s = get_state();
s->info.method = "GET";
s->op = OP_GET;
std::string uri = make_uri(bucket_name, obj_name);
s->relative_uri = uri;
s->info.request_uri = uri;
s->info.effective_uri = uri;
s->info.request_params = "";
s->info.domain = ""; /* XXX ? */
return 0;
}
virtual int get_params() {
return 0;
}
virtual void send_response() {}
}; /* RGWGetAttrsRequest */
class RGWSetAttrsRequest : public RGWLibRequest,
public RGWSetAttrs /* RGWOp */
{
@ -2676,6 +2750,10 @@ public:
op = this;
}
const std::map<std::string, buffer::list>& get_attrs() {
return attrs;
}
bool only_bucket() override { return false; }
int op_init() override {
@ -2714,6 +2792,62 @@ public:
}; /* RGWSetAttrsRequest */
class RGWRMAttrsRequest : public RGWLibRequest,
public RGWRMAttrs /* RGWOp */
{
public:
const std::string& bucket_name;
const std::string& obj_name;
RGWRMAttrsRequest(CephContext* _cct,
std::unique_ptr<rgw::sal::RGWUser> _user,
const std::string& _bname, const std::string& _oname)
: RGWLibRequest(_cct, std::move(_user)), RGWRMAttrs(),
bucket_name(_bname), obj_name(_oname) {
op = this;
}
const rgw::sal::RGWAttrs& get_attrs() {
return attrs;
}
virtual bool only_bucket() { return false; }
virtual int op_init() {
// assign store, s, and dialect_handler
RGWObjectCtx* rados_ctx
= static_cast<RGWObjectCtx*>(get_state()->obj_ctx);
// framework promises to call op_init after parent init
assert(rados_ctx);
RGWOp::init(rados_ctx->get_store(), get_state(), this);
op = this; // assign self as op: REQUIRED
return 0;
}
virtual int header_init() {
struct req_state* s = get_state();
s->info.method = "DELETE";
s->op = OP_PUT;
std::string uri = make_uri(bucket_name, obj_name);
s->relative_uri = uri;
s->info.request_uri = uri;
s->info.effective_uri = uri;
s->info.request_params = "";
s->info.domain = ""; /* XXX ? */
return 0;
}
virtual int get_params() {
return 0;
}
virtual void send_response() {}
}; /* RGWRMAttrsRequest */
/*
* Send request to get the rados cluster stats
*/

141
src/rgw/rgw_op.cc
View File

@ -1003,7 +1003,7 @@ void RGWGetObjTags::execute(optional_yield y)
s->object->set_atomic(s->obj_ctx);
op_ret = s->object->get_obj_attrs(s->obj_ctx, s->yield);
op_ret = s->object->get_obj_attrs(s->obj_ctx, y);
if (op_ret < 0) {
ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << s->object
<< " ret=" << op_ret << dendl;
@ -1052,7 +1052,7 @@ void RGWPutObjTags::execute(optional_yield y)
}
s->object->set_atomic(s->obj_ctx);
op_ret = s->object->modify_obj_attrs(s->obj_ctx, RGW_ATTR_TAGS, tags_bl, s->yield);
op_ret = s->object->modify_obj_attrs(s->obj_ctx, RGW_ATTR_TAGS, tags_bl, y);
if (op_ret == -ECANCELED){
op_ret = -ERR_TAG_CONFLICT;
}
@ -1092,7 +1092,7 @@ void RGWDeleteObjTags::execute(optional_yield y)
if (rgw::sal::RGWObject::empty(s->object.get()))
return;
op_ret = s->object->delete_obj_attrs(s->obj_ctx, RGW_ATTR_TAGS, s->yield);
op_ret = s->object->delete_obj_attrs(s->obj_ctx, RGW_ATTR_TAGS, y);
}
int RGWGetBucketTags::verify_permission(optional_yield y)
@ -1138,10 +1138,10 @@ void RGWPutBucketTags::execute(optional_yield y)
ldpp_dout(this, 0) << "forward_request_to_master returned ret=" << op_ret << dendl;
}
op_ret = retry_raced_bucket_write(s->bucket.get(), [this] {
op_ret = retry_raced_bucket_write(s->bucket.get(), [this, y] {
rgw::sal::RGWAttrs attrs = s->bucket->get_attrs();
attrs[RGW_ATTR_TAGS] = tags_bl;
return s->bucket->set_instance_attrs(attrs, s->yield);
return s->bucket->set_instance_attrs(attrs, y);
});
}
@ -1165,10 +1165,10 @@ void RGWDeleteBucketTags::execute(optional_yield y)
return;
}
op_ret = retry_raced_bucket_write(s->bucket.get(), [this] {
op_ret = retry_raced_bucket_write(s->bucket.get(), [this, y] {
rgw::sal::RGWAttrs attrs = s->bucket->get_attrs();
attrs.erase(RGW_ATTR_TAGS);
op_ret = s->bucket->set_instance_attrs(attrs, s->yield);
op_ret = s->bucket->set_instance_attrs(attrs, y);
if (op_ret < 0) {
ldpp_dout(this, 0) << "RGWDeleteBucketTags() failed to remove RGW_ATTR_TAGS on bucket="
<< s->bucket->get_name()
@ -2811,7 +2811,7 @@ void RGWListBucket::execute(optional_yield y)
rgw::sal::RGWBucket::ListResults results;
op_ret = s->bucket->list(params, max, results, s->yield);
op_ret = s->bucket->list(params, max, results, y);
if (op_ret >= 0) {
next_marker = results.next_marker;
is_truncated = results.is_truncated;
@ -3200,10 +3200,10 @@ void RGWCreateBucket::execute(optional_yield y)
}
op_ret = store->ctl()->bucket->link_bucket(s->user->get_id(), s->bucket->get_key(),
s->bucket->get_creation_time(), s->yield, false);
s->bucket->get_creation_time(), y, false);
if (op_ret && !existed && op_ret != -EEXIST) {
/* if it exists (or previously existed), don't remove it! */
op_ret = store->ctl()->bucket->unlink_bucket(s->user->get_id(), s->bucket->get_key(), s->yield);
op_ret = store->ctl()->bucket->unlink_bucket(s->user->get_id(), s->bucket->get_key(), y);
if (op_ret < 0) {
ldpp_dout(this, 0) << "WARNING: failed to unlink bucket: ret=" << op_ret
<< dendl;
@ -3221,7 +3221,7 @@ void RGWCreateBucket::execute(optional_yield y)
do {
map<string, bufferlist> battrs;
op_ret = s->bucket->get_bucket_info(s->yield);
op_ret = s->bucket->get_bucket_info(y);
if (op_ret < 0) {
return;
} else if (!s->bucket->is_owner(s->user.get())) {
@ -3258,7 +3258,7 @@ void RGWCreateBucket::execute(optional_yield y)
/* This will also set the quota on the bucket. */
op_ret = store->ctl()->bucket->set_bucket_instance_attrs(s->bucket->get_info(), attrs,
&s->bucket->get_info().objv_tracker,
s->yield);
y);
} while (op_ret == -ECANCELED && tries++ < 20);
/* Restore the proper return code. */
@ -3319,7 +3319,7 @@ void RGWDeleteBucket::execute(optional_yield y)
ldpp_dout(this, 1) << "WARNING: failed to sync user stats before bucket delete: op_ret= " << op_ret << dendl;
}
op_ret = s->bucket->check_empty(s->yield);
op_ret = s->bucket->check_empty(y);
if (op_ret < 0) {
return;
}
@ -3350,8 +3350,8 @@ void RGWDeleteBucket::execute(optional_yield y)
}
}
op_ret = s->bucket->remove_bucket(false, prefix, delimiter, false, nullptr, s->yield);
op_ret = s->bucket->remove_bucket(false, prefix, delimiter, false, nullptr,
y);
if (op_ret < 0 && op_ret == -ECANCELED) {
// lost a race, either with mdlog sync or another delete bucket operation.
// in either case, we've already called ctl.bucket->unlink_bucket()
@ -3406,13 +3406,13 @@ int RGWPutObj::init_processing(optional_yield y) {
}
std::unique_ptr<rgw::sal::RGWBucket> bucket;
ret = store->get_bucket(s->user.get(), copy_source_tenant_name, copy_source_bucket_name,
&bucket, s->yield);
&bucket, y);
if (ret < 0) {
ldpp_dout(this, 5) << __func__ << "(): get_bucket() returned ret=" << ret << dendl;
return ret;
}
ret = bucket->get_bucket_info(s->yield);
ret = bucket->get_bucket_info(y);
if (ret < 0) {
ldpp_dout(this, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl;
return ret;
@ -7267,6 +7267,98 @@ ssize_t RGWBulkUploadOp::AlignedStreamGetter::get_exactly(const size_t want,
return len;
}
int RGWGetAttrs::verify_permission(optional_yield y)
{
s->object->set_atomic(s->obj_ctx);
auto iam_action = s->object->get_instance().empty() ?
rgw::IAM::s3GetObject :
rgw::IAM::s3GetObjectVersion;
if (!verify_object_permission(this, s, iam_action)) {
return -EACCES;
}
return 0;
}
void RGWGetAttrs::pre_exec()
{
rgw_bucket_object_pre_exec(s);
}
void RGWGetAttrs::execute(optional_yield y)
{
op_ret = get_params();
if (op_ret < 0)
return;
s->object->set_atomic(s->obj_ctx);
op_ret = s->object->get_obj_attrs(s->obj_ctx, s->yield);
if (op_ret < 0) {
ldpp_dout(this, 0) << "ERROR: failed to get obj attrs, obj=" << s->object
<< " ret=" << op_ret << dendl;
return;
}
/* XXX RGWObject::get_obj_attrs() does not support filtering (yet) */
auto& obj_attrs = s->object->get_attrs();
if (attrs.size() != 0) {
/* return only attrs requested */
for (auto& att : attrs) {
auto iter = obj_attrs.find(att.first);
if (iter != obj_attrs.end()) {
att.second = iter->second;
}
}
} else {
/* return all attrs */
for (auto& att : obj_attrs) {
attrs.insert(get_attrs_t::value_type(att.first, att.second));;
}
}
return;
}
int RGWRMAttrs::verify_permission(optional_yield y)
{
// This looks to be part of the RGW-NFS machinery and has no S3 or
// Swift equivalent.
bool perm;
if (!rgw::sal::RGWObject::empty(s->object.get())) {
perm = verify_object_permission_no_policy(this, s, RGW_PERM_WRITE);
} else {
perm = verify_bucket_permission_no_policy(this, s, RGW_PERM_WRITE);
}
if (!perm)
return -EACCES;
return 0;
}
void RGWRMAttrs::pre_exec()
{
rgw_bucket_object_pre_exec(s);
}
void RGWRMAttrs::execute(optional_yield y)
{
op_ret = get_params();
if (op_ret < 0)
return;
s->object->set_atomic(s->obj_ctx);
op_ret = s->object->set_obj_attrs(s->obj_ctx, nullptr, &attrs, y);
if (op_ret < 0) {
ldpp_dout(this, 0) << "ERROR: failed to delete obj attrs, obj=" << s->object
<< " ret=" << op_ret << dendl;
}
return;
}
int RGWSetAttrs::verify_permission(optional_yield y)
{
// This looks to be part of the RGW-NFS machinery and has no S3 or
@ -7296,16 +7388,17 @@ void RGWSetAttrs::execute(optional_yield y)
if (!rgw::sal::RGWObject::empty(s->object.get())) {
rgw::sal::RGWAttrs a(attrs);
op_ret = s->object->set_obj_attrs(s->obj_ctx, &a, nullptr, s->yield);
op_ret = s->object->set_obj_attrs(s->obj_ctx, &a, nullptr, y);
} else {
for (auto& iter : attrs) {
s->bucket_attrs[iter.first] = std::move(iter.second);
}
op_ret = store->ctl()->bucket->set_bucket_instance_attrs(s->bucket->get_info(), attrs,
&s->bucket->get_info().objv_tracker,
s->yield);
op_ret = store->ctl()->bucket->set_bucket_instance_attrs(
s->bucket->get_info(), attrs, &s->bucket->get_info().objv_tracker,
s->yield);
}
}
} /* RGWSetAttrs::execute() */
void RGWGetObjLayout::pre_exec()
{
@ -7320,14 +7413,14 @@ void RGWGetObjLayout::execute(optional_yield y)
std::unique_ptr<rgw::sal::RGWObject::ReadOp> stat_op(s->object->get_read_op(s->obj_ctx));
op_ret = stat_op->prepare(s->yield);
op_ret = stat_op->prepare(y);
if (op_ret < 0) {
return;
}
head_obj = stat_op->result.head_obj;
op_ret = stat_op->get_manifest(&manifest, s->yield);
op_ret = stat_op->get_manifest(&manifest, y);
}

57
src/rgw/rgw_op.h
View File

@ -25,6 +25,7 @@
#include <boost/optional.hpp>
#include <boost/utility/in_place_factory.hpp>
#include <boost/function.hpp>
#include <boost/container/flat_map.hpp>
#include "common/armor.h"
#include "common/mime.h"
@ -2129,9 +2130,38 @@ inline void complete_etag(MD5& hash, string *etag)
*etag = etag_buf_str;
} /* complete_etag */
using boost::container::flat_map;
class RGWGetAttrs : public RGWOp {
public:
using get_attrs_t = flat_map<std::string, std::optional<buffer::list>>;
protected:
get_attrs_t attrs;
public:
RGWGetAttrs()
{}
virtual ~RGWGetAttrs() {}
void emplace_key(std::string&& key) {
attrs.emplace(std::move(key), std::nullopt);
}
int verify_permission(optional_yield y);
void pre_exec();
void execute(optional_yield y);
virtual int get_params() = 0;
virtual void send_response() = 0;
virtual const char* name() const { return "get_attrs"; }
virtual RGWOpType get_type() { return RGW_OP_GET_ATTRS; }
virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
}; /* RGWGetAttrs */
class RGWSetAttrs : public RGWOp {
protected:
map<string, buffer::list> attrs;
map<std::string, buffer::list> attrs;
public:
RGWSetAttrs() {}
@ -2152,6 +2182,31 @@ public:
uint32_t op_mask() override { return RGW_OP_TYPE_WRITE; }
};
class RGWRMAttrs : public RGWOp {
protected:
rgw::sal::RGWAttrs attrs;
public:
RGWRMAttrs()
{}
virtual ~RGWRMAttrs() {}
void emplace_key(std::string&& key) {
attrs.emplace(std::move(key), buffer::list());
}
int verify_permission(optional_yield y);
void pre_exec();
void execute(optional_yield y);
virtual int get_params() = 0;
virtual void send_response() = 0;
virtual const char* name() const { return "rm_attrs"; }
virtual RGWOpType get_type() { return RGW_OP_DELETE_ATTRS; }
virtual uint32_t op_mask() { return RGW_OP_TYPE_DELETE; }
}; /* RGWRMAttrs */
class RGWGetObjLayout : public RGWOp {
protected:
RGWObjManifest *manifest{nullptr};

3
src/rgw/rgw_op_type.h
View File

@ -42,6 +42,9 @@ enum RGWOpType {
RGW_OP_LIST_BUCKET_MULTIPARTS,
RGW_OP_DELETE_MULTI_OBJ,
RGW_OP_BULK_DELETE,
RGW_OP_GET_KEYS,
RGW_OP_GET_ATTRS,
RGW_OP_DELETE_ATTRS,
RGW_OP_SET_ATTRS,
RGW_OP_GET_CROSS_DOMAIN_POLICY,
RGW_OP_GET_HEALTH_CHECK,

14
src/test/CMakeLists.txt
View File

@ -320,6 +320,20 @@ target_link_libraries(ceph_test_librgw_file_marker
)
target_link_libraries(ceph_test_librgw_file_marker spawn)
# ceph_test_librgw_file_xattr (attribute ops)
add_executable(ceph_test_librgw_file_xattr
librgw_file_xattr.cc
)
target_include_directories(ceph_test_librgw_file_xattr SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw")
target_link_libraries(ceph_test_librgw_file_xattr
rgw
librados
ceph-common
${UNITTEST_LIBS}
${EXTRALIBS}
)
target_link_libraries(ceph_test_librgw_file_xattr spawn)
# ceph_test_rgw_token
add_executable(ceph_test_rgw_token
test_rgw_token.cc

434
src/test/librgw_file_xattr.cc Normal file
View File

@ -0,0 +1,434 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2015 Red Hat, Inc.
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#include <stdint.h>
#include <tuple>
#include <iostream>
#include <vector>
#include <map>
#include <random>
#include <boost/algorithm/string.hpp>
#include "xxhash.h"
#include "include/rados/librgw.h"
#include "include/rados/rgw_file.h"
#include "rgw/rgw_file.h"
#include "gtest/gtest.h"
#include "common/ceph_argparse.h"
#include "common/errno.h"
#include "common/debug.h"
#include "global/global_init.h"
#include "include/ceph_assert.h"
#define dout_context g_ceph_context
#define dout_subsys ceph_subsys_rgw
namespace {
using namespace rgw;
string uid("testuser");
string access_key("");
string secret_key("");
librgw_t rgw_h = nullptr;
struct rgw_fs *fs = nullptr;
uint32_t owner_uid = 867;
uint32_t owner_gid = 5309;
uint32_t create_mask = RGW_SETATTR_UID | RGW_SETATTR_GID | RGW_SETATTR_MODE;
string bucket_name{"v4recov"};
string object_path{"node0/clientids"};
string key1{"black"};
string val1{"metallic"};
struct rgw_file_handle *bucket_fh = nullptr;
struct rgw_file_handle *object_fh = nullptr;
typedef std::tuple<string,uint64_t, struct rgw_file_handle*> fid_type;
std::vector<fid_type> fids;
class obj_rec
{
public:
string name;
struct rgw_file_handle* fh;
struct rgw_file_handle* parent_fh;
RGWFileHandle* rgw_fh; // alias into fh
struct state {
bool readdir;
state() : readdir(false) {}
} state;
obj_rec(string _name, struct rgw_file_handle* _fh,
struct rgw_file_handle* _parent_fh, RGWFileHandle* _rgw_fh)
: name(std::move(_name)), fh(_fh), parent_fh(_parent_fh),
rgw_fh(_rgw_fh) {}
void clear() {
fh = nullptr;
rgw_fh = nullptr;
}
void sync() {
if (fh)
rgw_fh = get_rgwfh(fh);
}
friend ostream& operator<<(ostream& os, const obj_rec& rec);
};
ostream& operator<<(ostream& os, const obj_rec& rec)
{
RGWFileHandle* rgw_fh = rec.rgw_fh;
if (rgw_fh) {
const char* type = rgw_fh->is_dir() ? "DIR " : "FILE ";
os << rec.rgw_fh->full_object_name()
<< " (" << rec.rgw_fh->object_name() << "): "
<< type;
}
return os;
}
typedef std::vector<obj_rec> obj_vec;
obj_vec ovec;
bool do_stat = false;
bool do_create = false;
bool do_delete = false;
bool do_hexdump = false;
bool verbose = false;
struct {
int argc;
char **argv;
} saved_args;
}
TEST(LibRGW, INIT) {
int ret = librgw_create(&rgw_h, saved_args.argc, saved_args.argv);
ASSERT_EQ(ret, 0);
ASSERT_NE(rgw_h, nullptr);
}
TEST(LibRGW, MOUNT) {
int ret = rgw_mount(rgw_h, uid.c_str(), access_key.c_str(),
secret_key.c_str(), &fs, RGW_MOUNT_FLAG_NONE);
ASSERT_EQ(ret, 0);
ASSERT_NE(fs, nullptr);
}
TEST(LibRGW, LOOKUP_BUCKET) {
int ret = rgw_lookup(fs, fs->root_fh, bucket_name.c_str(), &bucket_fh,
nullptr, 0, RGW_LOOKUP_FLAG_NONE);
ASSERT_EQ(ret, 0);
}
TEST(LibRGW, CREATE_BUCKET) {
if ((! bucket_fh) && do_create) {
struct stat st;
st.st_uid = owner_uid;
st.st_gid = owner_gid;
st.st_mode = 755;
int ret = rgw_mkdir(fs, fs->root_fh, bucket_name.c_str(), &st, create_mask,
&bucket_fh, RGW_MKDIR_FLAG_NONE);
ASSERT_EQ(ret, 0);
}
}
TEST(LibRGW, CREATE_PATH) {
if (!bucket_fh)
return;
vector<string> segs;
boost::split(segs, object_path, boost::is_any_of("/"));
struct stat st;
st.st_uid = owner_uid;
st.st_gid = owner_gid;
st.st_mode = 755;
int ix, ret, sz = segs.size();
for (ix = 0; ix < sz; ++ix) {
auto& seg = segs[ix];
struct rgw_file_handle* parent_fh = (ix > 0) ? ovec[ix-1].fh : bucket_fh;
obj_rec dir{seg, nullptr, parent_fh, nullptr};
if (do_create) {
ret = rgw_mkdir(fs, dir.parent_fh, dir.name.c_str(), &st, create_mask,
&dir.fh, RGW_MKDIR_FLAG_NONE);
} else {
ret = rgw_lookup(fs, dir.parent_fh, dir.name.c_str(), &dir.fh,
nullptr, 0, RGW_LOOKUP_FLAG_NONE);
}
ASSERT_EQ(ret, 0);
dir.sync();
ovec.push_back(dir);
if (verbose) {
std::cout << "create: " << dir.name << std::endl;
}
}
}
TEST(LibRGW, CHECK_PATH_REFS) {
if (!bucket_fh)
return;
int ix, sz = ovec.size();
for (ix = 0; ix < sz; ++ix) {
auto& dir = ovec[ix];
if (verbose) {
std::cout << "name: " << dir.name
<< " refcnt: " << dir.rgw_fh->get_refcnt()
<< std::endl;
}
if (ix == 0) {
// sentinel, +1 parent, +1 path
ASSERT_EQ(dir.rgw_fh->get_refcnt(), 3U);
}
if (ix == 1) {
// sentinel, +1 path
ASSERT_EQ(dir.rgw_fh->get_refcnt(), 2U);
}
}
}
TEST(LibRGW, SETXATTR1) {
if (!bucket_fh)
return;
auto& dir = ovec[ovec.size()-1];
rgw_xattrstr xattr_k = { const_cast<char*>(key1.c_str()),
uint32_t(key1.length()) };
rgw_xattrstr xattr_v = { const_cast<char*>(val1.c_str()),
uint32_t(val1.length()) };
rgw_xattr xattr = { xattr_k, xattr_v };
rgw_xattrlist xattrlist = { &xattr, 1 };
int ret = rgw_setxattrs(fs, dir.fh, &xattrlist, RGW_SETXATTR_FLAG_NONE);
ASSERT_EQ(ret, 0);
}
extern "C" {
static int getattr_cb(rgw_xattrlist *attrs, void *arg, uint32_t flags)
{
auto& attrmap =
*(static_cast<std::map<std::string, std::string>*>(arg));
for (uint32_t ix = 0; ix < attrs->xattr_cnt; ++ix) {
auto& xattr = attrs->xattrs[ix];
string k{xattr.key.val, xattr.key.len};
string v{xattr.val.val, xattr.val.len};
if (verbose) {
std::cout << __func__
<< " attr k: " << k << " v: " << v
<< std::endl;
}
attrmap.insert(std::map<std::string, std::string>::value_type(k, v));
}
return 0;
}
}
TEST(LibRGW, GETXATTR1) {
if (!bucket_fh)
return;
using std::get;
auto& dir = ovec[ovec.size()-1];
rgw_xattrstr xattr_k1 =
{const_cast<char*>(key1.c_str()), uint32_t(key1.length())};
rgw_xattrstr xattr_v1 = {nullptr, 0};
std::string key2 = "user.rgw.etag";
rgw_xattrstr xattr_k2 =
{const_cast<char*>(key2.c_str()), uint32_t(key2.length())};
rgw_xattrstr xattr_v2 = {nullptr, 0};
rgw_xattr xattrs[2] = {{xattr_k1, xattr_v1},
{xattr_k2, xattr_v2}};
/* XXX gcc won't accept static_cast here, don't see why */
rgw_xattrlist xattrlist = {reinterpret_cast<rgw_xattr*>(&xattrs), 2};
std::map<std::string, std::string> out_attrmap;
int ret = rgw_getxattrs(fs, dir.fh, &xattrlist, getattr_cb, &out_attrmap,
RGW_GETXATTR_FLAG_NONE);
ASSERT_EQ(ret, 0);
/* check exposed attrs */
ASSERT_TRUE(out_attrmap.find("user.rgw.etag") != out_attrmap.end());
/* check our user attr */
ASSERT_TRUE(out_attrmap.find(key1) != out_attrmap.end());
}
TEST(LibRGW, LSXATTR1) {
if (!bucket_fh)
return;
using std::get;
auto& dir = ovec[ovec.size()-1];
rgw_xattrstr filter_prefix = { nullptr, 0}; // XXX ignored, for now
std::map<std::string, std::string> out_attrmap;
int ret = rgw_lsxattrs(fs, dir.fh, &filter_prefix, getattr_cb,
&out_attrmap, RGW_LSXATTR_FLAG_NONE);
ASSERT_EQ(ret, 0);
/* check exposed attrs */
ASSERT_TRUE(out_attrmap.find("user.rgw.etag") != out_attrmap.end());
}
TEST(LibRGW, RMXATTR1) {
if (!bucket_fh)
return;
using std::get;
auto& dir = ovec[ovec.size()-1];
rgw_xattrstr xattr_k = { const_cast<char*>(key1.c_str()),
uint32_t(key1.length()) };
rgw_xattrstr xattr_v = { nullptr, 0 };
rgw_xattr xattr = { xattr_k, xattr_v };
rgw_xattrlist xattrlist = { &xattr, 1 };
int ret = rgw_rmxattrs(fs, dir.fh, &xattrlist, RGW_RMXATTR_FLAG_NONE);
ASSERT_EQ(ret, 0);
}
TEST(LibRGW, LSXATTR2) {
if (!bucket_fh)
return;
using std::get;
auto& dir = ovec[ovec.size()-1];
rgw_xattrstr filter_prefix = { nullptr, 0 }; // XXX ignored, for now
std::map<std::string, std::string> out_attrmap;
int ret = rgw_lsxattrs(fs, dir.fh, &filter_prefix, getattr_cb,
&out_attrmap, RGW_LSXATTR_FLAG_NONE);
ASSERT_EQ(ret, 0);
/* check exposed attrs */
ASSERT_TRUE(out_attrmap.find("user.rgw.etag") != out_attrmap.end());
/* verify deletion */
ASSERT_TRUE(out_attrmap.find(key1) == out_attrmap.end());
}
TEST(LibRGW, CLEANUP) {
int ret = 0;
if (object_fh) {
ret = rgw_fh_rele(fs, object_fh, 0 /* flags */);
ASSERT_EQ(ret, 0);
}
if (bucket_fh) {
ret = rgw_fh_rele(fs, bucket_fh, 0 /* flags */);
}
ASSERT_EQ(ret, 0);
}
TEST(LibRGW, UMOUNT) {
if (! fs)
return;
int ret = rgw_umount(fs, RGW_UMOUNT_FLAG_NONE);
ASSERT_EQ(ret, 0);
}
TEST(LibRGW, SHUTDOWN) {
librgw_shutdown(rgw_h);
}
int main(int argc, char *argv[])
{
char *v{nullptr};
string val;
vector<const char*> args;
argv_to_vec(argc, const_cast<const char**>(argv), args);
env_to_vec(args);
v = getenv("AWS_ACCESS_KEY_ID");
if (v) {
access_key = v;
}
v = getenv("AWS_SECRET_ACCESS_KEY");
if (v) {
secret_key = v;
}
for (auto arg_iter = args.begin(); arg_iter != args.end();) {
if (ceph_argparse_witharg(args, arg_iter, &val, "--access",
(char*) nullptr)) {
access_key = val;
} else if (ceph_argparse_witharg(args, arg_iter, &val, "--secret",
(char*) nullptr)) {
secret_key = val;
} else if (ceph_argparse_witharg(args, arg_iter, &val, "--uid",
(char*) nullptr)) {
uid = val;
} else if (ceph_argparse_witharg(args, arg_iter, &val, "--bn",
(char*) nullptr)) {
bucket_name = val;
} else if (ceph_argparse_flag(args, arg_iter, "--stat",
(char*) nullptr)) {
do_stat = true;
} else if (ceph_argparse_flag(args, arg_iter, "--create",
(char*) nullptr)) {
do_create = true;
} else if (ceph_argparse_flag(args, arg_iter, "--delete",
(char*) nullptr)) {
do_delete = true;
} else if (ceph_argparse_flag(args, arg_iter, "--hexdump",
(char*) nullptr)) {
do_hexdump = true;
} else if (ceph_argparse_flag(args, arg_iter, "--verbose",
(char*) nullptr)) {
verbose = true;
} else {
++arg_iter;
}
}
/* dont accidentally run as anonymous */
if ((access_key == "") ||
(secret_key == "")) {
std::cout << argv[0] << " no AWS credentials, exiting" << std::endl;
return EPERM;
}
saved_args.argc = argc;
saved_args.argv = argv;
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}