RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-24 03:27:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #43844 from dang/wip-dang-zipper-doc

Browse Source 
RGW Zipper - Cleanup and API doc pass
This commit is contained in:
Daniel Gryniewicz 2021-11-09 13:05:14 -05:00 committed by GitHub
commit eebd4c3cf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 964 additions and 619 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/rgw/rgw_bucket.cc
View File

@ -936,7 +936,19 @@ int RGWBucketAdminOp::link(rgw::sal::Store* store, RGWBucketAdminOpState& op_sta
if (*loc_bucket != *old_bucket) {
// like RGWRados::delete_bucket -- excepting no bucket_index work.
r = old_bucket->remove_metadata(dpp, &ep_data.ep_objv, null_yield);
r = static_cast<rgw::sal::RadosStore*>(store)->ctl()->bucket->remove_bucket_entrypoint_info(
old_bucket->get_key(), null_yield, dpp,
RGWBucketCtl::Bucket::RemoveParams()
.set_objv_tracker(&ep_data.ep_objv));
if (r < 0) {
set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name());
return r;
}
r = static_cast<rgw::sal::RadosStore*>(store)->ctl()->bucket->remove_bucket_instance_info(
old_bucket->get_key(), old_bucket->get_info(),
null_yield, dpp,
RGWBucketCtl::BucketInstance::RemoveParams()
.set_objv_tracker(&ep_data.ep_objv));
if (r < 0) {
set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name());
return r;
@ -1053,7 +1065,7 @@ static int bucket_stats(rgw::sal::Store* store,
string bucket_ver, master_ver;
string max_marker;
ret = bucket->get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker);
ret = bucket->read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0) {
cerr << "error getting bucket stats bucket=" << bucket->get_name() << " ret=" << ret << std::endl;
return ret;
@ -1153,14 +1165,14 @@ int RGWBucketAdminOp::limit_check(rgw::sal::Store* store,
* as we may now not reach the end of
* the loop body */
ret = bucket->get_bucket_info(dpp, null_yield);
ret = bucket->load_bucket(dpp, null_yield);
if (ret < 0)
continue;
/* need stats for num_entries */
string bucket_ver, master_ver;
std::map<RGWObjCategory, RGWStorageStats> stats;
ret = bucket->get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, nullptr);
ret = bucket->read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, nullptr);
if (ret < 0)
continue;

6
src/rgw/rgw_lc.cc
View File

@ -844,7 +844,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target,
auto& [rule, obj] = wt;
if (obj_has_expired(this, cct, obj.meta.mtime, rule.mp_expiration)) {
rgw_obj_key key(obj.key);
std::unique_ptr<rgw::sal::MultipartUpload> mpu = store->get_multipart_upload(target, key.name);
std::unique_ptr<rgw::sal::MultipartUpload> mpu = target->get_multipart_upload(key.name);
RGWObjectCtx rctx(store);
int ret = mpu->abort(this, cct, &rctx);
if (ret == 0) {
@ -1446,9 +1446,9 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker,
return ret;
}
ret = bucket->get_bucket_info(this, null_yield);
ret = bucket->load_bucket(this, null_yield);
if (ret < 0) {
ldpp_dout(this, 0) << "LC:get_bucket_info for " << bucket_name
ldpp_dout(this, 0) << "LC:load_bucket for " << bucket_name
<< " failed" << dendl;
return ret;
}

29
src/rgw/rgw_op.cc
View File

@ -389,7 +389,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp,
if (!upload_id.empty() && !copy_src) {
/* multipart upload */
std::unique_ptr<rgw::sal::MultipartUpload> upload;
upload = store->get_multipart_upload(bucket, object->get_name(), upload_id);
upload = bucket->get_multipart_upload(object->get_name(), upload_id);
mpobj = upload->get_meta_obj();
mpobj->set_in_extra_data(true);
object = mpobj.get();
@ -1900,7 +1900,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y)
}
s->object->set_obj_size(s->obj_size);
r = RGWRados::Object::Read::range_to_ofs(s->obj_size, ofs, end);
r = s->object->range_to_ofs(s->obj_size, ofs, end);
if (r < 0) {
return r;
}
@ -2049,7 +2049,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y)
s->object->set_obj_size(slo_info.total_size);
ldpp_dout(this, 20) << "s->obj_size=" << s->obj_size << dendl;
int r = RGWRados::Object::Read::range_to_ofs(total_len, ofs, end);
int r = s->object->range_to_ofs(total_len, ofs, end);
if (r < 0) {
return r;
}
@ -2827,7 +2827,6 @@ void RGWStatBucket::execute(optional_yield y)
if (op_ret) {
return;
}
op_ret = bucket->update_container_stats(s);
}
int RGWListBucket::verify_permission(optional_yield y)
@ -3276,7 +3275,7 @@ void RGWCreateBucket::execute(optional_yield y)
/* We're replacing bucket with the newly created one */
ldpp_dout(this, 10) << "user=" << s->user << " bucket=" << tmp_bucket << dendl;
op_ret = store->create_bucket(this, s->user.get(), tmp_bucket, zonegroup_id,
op_ret = s->user->create_bucket(this, tmp_bucket, zonegroup_id,
placement_rule,
info.swift_ver_location,
pquota_info, policy, attrs, info, ep_objv,
@ -3300,7 +3299,7 @@ void RGWCreateBucket::execute(optional_yield y)
do {
map<string, bufferlist> battrs;
op_ret = s->bucket->get_bucket_info(this, y);
op_ret = s->bucket->load_bucket(this, y);
if (op_ret < 0) {
return;
} else if (!s->bucket->is_owner(s->user.get())) {
@ -3492,9 +3491,9 @@ int RGWPutObj::init_processing(optional_yield y) {
return ret;
}
ret = bucket->get_bucket_info(this, y);
ret = bucket->load_bucket(this, y);
if (ret < 0) {
ldpp_dout(this, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl;
ldpp_dout(this, 5) << __func__ << "(): load_bucket() returned ret=" << ret << dendl;
return ret;
}
copy_source_bucket_info = bucket->get_info();
@ -3926,7 +3925,7 @@ void RGWPutObj::execute(optional_yield y)
if (multipart) {
s->trace->SetTag(tracing::UPLOAD_ID, multipart_upload_id);
std::unique_ptr<rgw::sal::MultipartUpload> upload;
upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(),
upload = s->bucket->get_multipart_upload(s->object->get_name(),
multipart_upload_id);
op_ret = upload->get_info(this, s->yield, s->obj_ctx, &pdest_placement);
if (op_ret < 0) {
@ -6151,7 +6150,7 @@ void RGWInitMultipart::execute(optional_yield y)
}
std::unique_ptr<rgw::sal::MultipartUpload> upload;
upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(),
upload = s->bucket->get_multipart_upload(s->object->get_name(),
upload_id);
op_ret = upload->init(this, s->yield, s->obj_ctx, s->owner, s->dest_placement, attrs);
@ -6278,7 +6277,7 @@ void RGWCompleteMultipart::execute(optional_yield y)
return;
}
upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id);
upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id);
s->trace->SetTag(tracing::UPLOAD_ID, upload_id);
@ -6495,7 +6494,7 @@ void RGWAbortMultipart::execute(optional_yield y)
if (upload_id.empty() || rgw::sal::Object::empty(s->object.get()))
return;
upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id);
upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id);
RGWObjectCtx *obj_ctx = static_cast<RGWObjectCtx *>(s->obj_ctx);
op_ret = upload->abort(this, s->cct, obj_ctx);
}
@ -6523,7 +6522,7 @@ void RGWListMultipart::execute(optional_yield y)
if (op_ret < 0)
return;
upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id);
upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id);
rgw::sal::Attrs attrs;
op_ret = upload->get_info(this, s->yield, s->obj_ctx, nullptr, &attrs);
@ -6961,7 +6960,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie
goto binfo_fail;
}
ret = bucket->get_bucket_info(dpp, s->yield);
ret = bucket->load_bucket(dpp, s->yield);
if (ret < 0) {
goto binfo_fail;
}
@ -7241,7 +7240,7 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y)
placement_rule.storage_class = s->info.storage_class;
forward_req_info(this, s->cct, info, bucket_name);
op_ret = store->create_bucket(this, s->user.get(), new_bucket,
op_ret = s->user->create_bucket(this, new_bucket,
store->get_zone()->get_zonegroup().get_id(),
placement_rule, swift_ver_location,
pquota_info, policy, attrs,

2
src/rgw/rgw_op.h
View File

@ -855,7 +855,7 @@ protected:
std::map<std::string, bool> categories;
std::map<rgw_user_bucket, rgw_usage_log_entry> usage;
std::map<std::string, rgw_usage_log_entry> summary_map;
std::map<std::string, cls_user_bucket_entry> buckets_usage;
std::map<std::string, bucket_meta_entry> buckets_usage;
cls_user_header header;
RGWStorageStats stats;
public:

6
src/rgw/rgw_quota.cc
View File

@ -273,11 +273,11 @@ int BucketAsyncRefreshHandler::init_fetch()
ldpp_dout(&dp, 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
r = rbucket->get_bucket_stats_async(&dp, RGW_NO_SHARD, this);
r = rbucket->read_stats_async(&dp, RGW_NO_SHARD, this);
if (r < 0) {
ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
/* get_bucket_stats_async() dropped our reference already */
/* read_stats_async() dropped our reference already */
return r;
}
@ -345,7 +345,7 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_
string master_ver;
map<RGWObjCategory, RGWStorageStats> bucket_stats;
r = bucket->get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, bucket_stats);
r = bucket->read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, bucket_stats);
if (r < 0) {
ldpp_dout(dpp, 0) << "could not get bucket stats for bucket="
<< _b.name << dendl;

21
src/rgw/rgw_rados.cc
View File

@ -3643,8 +3643,27 @@ static void set_copy_attrs(map<string, bufferlist>& src_attrs,
int RGWRados::rewrite_obj(rgw::sal::Object* obj, const DoutPrefixProvider *dpp, optional_yield y)
{
RGWObjectCtx rctx(this->store);
rgw::sal::Attrs attrset;
uint64_t obj_size;
ceph::real_time mtime;
RGWRados::Object op_target(this, obj->get_bucket()->get_info(), rctx, obj->get_obj());
RGWRados::Object::Read read_op(&op_target);
return obj->copy_obj_data(rctx, obj->get_bucket(), obj, 0, NULL, dpp, y);
read_op.params.attrs = &attrset;
read_op.params.obj_size = &obj_size;
read_op.params.lastmod = &mtime;
int ret = read_op.prepare(y, dpp);
if (ret < 0)
return ret;
attrset.erase(RGW_ATTR_ID_TAG);
attrset.erase(RGW_ATTR_TAIL_TAG);
return store->getRados()->copy_obj_data(rctx, obj->get_bucket(),
obj->get_bucket()->get_info().placement_rule,
read_op, obj_size - 1, obj, NULL, mtime,
attrset, 0, real_time(), NULL, dpp, y);
}
struct obj_time_weight {

2
src/rgw/rgw_rest.cc
View File

@ -1615,7 +1615,7 @@ int RGWListBucketMultiparts_ObjStore::get_params(optional_yield y)
string upload_id_marker = s->info.args.get("upload-id-marker");
if (!key_marker.empty()) {
std::unique_ptr<rgw::sal::MultipartUpload> upload;
upload = store->get_multipart_upload(s->bucket.get(), key_marker,
upload = s->bucket->get_multipart_upload(key_marker,
upload_id_marker);
marker_meta = upload->get_meta();
marker_key = upload->get_key();

2
src/rgw/rgw_rest_log.cc
View File

@ -484,7 +484,7 @@ void RGWOp_BILog_Info::execute(optional_yield y) {
}
map<RGWObjCategory, RGWStorageStats> stats;
int ret = bucket->get_bucket_stats(s, shard_id, &bucket_ver, &master_ver, stats, &max_marker, &syncstopped);
int ret = bucket->read_stats(s, shard_id, &bucket_ver, &master_ver, stats, &max_marker, &syncstopped);
if (ret < 0 && ret != -ENOENT) {
op_ret = ret;
return;

8
src/rgw/rgw_rest_s3.cc
View File

@ -1328,7 +1328,7 @@ static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log
formatter->close_section(); // Category
}
static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const cls_user_bucket_entry& entry)
static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const bucket_meta_entry& entry)
{
formatter->open_object_section("Entry");
encode_json("Bucket", name, formatter);
@ -1441,7 +1441,7 @@ void RGWGetUsage_ObjStore_S3::send_response()
formatter->open_object_section("User");
formatter->open_array_section("Buckets");
for (const auto& biter : buckets_usage) {
const cls_user_bucket_entry& entry = biter.second;
const bucket_meta_entry& entry = biter.second;
dump_usage_bucket_info(formatter, biter.first, entry);
}
formatter->close_section(); // Buckets
@ -2581,7 +2581,7 @@ int RGWPutObj_ObjStore_S3::get_encrypt_filter(
int res = 0;
if (!multipart_upload_id.empty()) {
std::unique_ptr<rgw::sal::MultipartUpload> upload =
store->get_multipart_upload(s->bucket.get(), s->object->get_name(),
s->bucket->get_multipart_upload(s->object->get_name(),
multipart_upload_id);
std::unique_ptr<rgw::sal::Object> obj = upload->get_meta_obj();
obj->set_in_extra_data(true);
@ -4023,7 +4023,7 @@ void RGWGetObjLayout_ObjStore_S3::send_response()
}
f.open_object_section("result");
s->object->get_obj_layout(this, s->yield, &f, s->obj_ctx);
s->object->dump_obj_layout(this, s->yield, &f, s->obj_ctx);
f.close_section();
rgw_flush_formatter(s, &f);
}

715
src/rgw/rgw_sal.h
View File

File diff suppressed because it is too large Load Diff

254
src/rgw/rgw_sal_dbstore.cc
View File

@ -52,10 +52,103 @@ namespace rgw::sal {
return 0;
}
Bucket* DBUser::create_bucket(rgw_bucket& bucket,
ceph::real_time creation_time)
int DBUser::create_bucket(const DoutPrefixProvider *dpp,
const rgw_bucket& b,
const string& zonegroup_id,
rgw_placement_rule& placement_rule,
string& swift_ver_location,
const RGWQuotaInfo * pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool *existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket_out,
optional_yield y)
{
return NULL;
int ret;
bufferlist in_data;
RGWBucketInfo master_info;
rgw_bucket *pmaster_bucket = nullptr;
uint32_t *pmaster_num_shards = nullptr;
real_time creation_time;
std::unique_ptr<Bucket> bucket;
obj_version objv, *pobjv = NULL;
/* If it exists, look it up; otherwise create it */
ret = store->get_bucket(dpp, this, b, &bucket, y);
if (ret < 0 && ret != -ENOENT)
return ret;
if (ret != -ENOENT) {
RGWAccessControlPolicy old_policy(store->ctx());
*existed = true;
if (swift_ver_location.empty()) {
swift_ver_location = bucket->get_info().swift_ver_location;
}
placement_rule.inherit_from(bucket->get_info().placement_rule);
// don't allow changes to the acl policy
/* int r = rgw_op_get_bucket_policy_from_attr(dpp, this, this, bucket->get_attrs(),
&old_policy, y);
if (r >= 0 && old_policy != policy) {
bucket_out->swap(bucket);
return -EEXIST;
}*/
} else {
bucket = std::make_unique<DBBucket>(store, b, this);
*existed = false;
bucket->set_attrs(attrs);
// XXX: For now single default zone and STANDARD storage class
// supported.
placement_rule.name = "default";
placement_rule.storage_class = "STANDARD";
}
/*
* XXX: If not master zone, fwd the request to master zone.
* For now DBStore has single zone.
*/
std::string zid = zonegroup_id;
/* if (zid.empty()) {
zid = svc()->zone->get_zonegroup().get_id();
} */
if (*existed) {
rgw_placement_rule selected_placement_rule;
/* XXX: Handle this when zone is implemented
ret = svc()->zone->select_bucket_placement(this.get_info(),
zid, placement_rule,
&selected_placement_rule, nullptr, y);
if (selected_placement_rule != info.placement_rule) {
ret = -EEXIST;
bucket_out->swap(bucket);
return ret;
} */
} else {
/* XXX: We may not need to send all these params. Cleanup the unused ones */
ret = store->getDB()->create_bucket(dpp, this->get_info(), bucket->get_key(),
zid, placement_rule, swift_ver_location, pquota_info,
attrs, info, pobjv, &ep_objv, creation_time,
pmaster_bucket, pmaster_num_shards, y, exclusive);
if (ret == -EEXIST) {
*existed = true;
ret = 0;
} else if (ret != 0) {
return ret;
}
}
bucket->set_version(ep_objv);
bucket->get_info() = info;
bucket_out->swap(bucket);
return ret;
}
int DBUser::read_attrs(const DoutPrefixProvider* dpp, optional_yield y)
@ -129,7 +222,7 @@ namespace rgw::sal {
{
int ret;
ret = get_bucket_info(dpp, y);
ret = load_bucket(dpp, y);
if (ret < 0)
return ret;
@ -147,7 +240,7 @@ namespace rgw::sal {
return 0;
}
int DBBucket::get_bucket_info(const DoutPrefixProvider *dpp, optional_yield y)
int DBBucket::load_bucket(const DoutPrefixProvider *dpp, optional_yield y)
{
int ret = 0;
@ -158,7 +251,7 @@ namespace rgw::sal {
}
/* stats - Not for first pass */
int DBBucket::get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id,
int DBBucket::read_stats(const DoutPrefixProvider *dpp, int shard_id,
std::string *bucket_ver, std::string *master_ver,
std::map<RGWObjCategory, RGWStorageStats>& stats,
std::string *max_marker, bool *syncstopped)
@ -166,12 +259,7 @@ namespace rgw::sal {
return 0;
}
int DBBucket::get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB *ctx)
{
return 0;
}
int DBBucket::read_bucket_stats(const DoutPrefixProvider *dpp, optional_yield y)
int DBBucket::read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB *ctx)
{
return 0;
}
@ -211,16 +299,6 @@ namespace rgw::sal {
}
int DBBucket::remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y)
{
/* XXX: same as DBBUcket::remove_bucket() but should return error if there are objects
* in that bucket. */
int ret = store->getDB()->remove_bucket(dpp, info);
return ret;
}
/* Make sure to call get_bucket_info() if you need it first */
bool DBBucket::is_owner(User* user)
{
@ -364,6 +442,13 @@ namespace rgw::sal {
return ret;
}
std::unique_ptr<MultipartUpload> DBBucket::get_multipart_upload(
const std::string& oid,
std::optional<std::string> upload_id,
ACLOwner owner, ceph::real_time mtime) {
return nullptr;
}
int DBBucket::list_multiparts(const DoutPrefixProvider *dpp,
const string& prefix,
string& marker,
@ -502,16 +587,6 @@ namespace rgw::sal {
return set_obj_attrs(dpp, rctx, nullptr, &rmattr, y, &target);
}
int DBObject::copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket,
Object* dest_obj,
uint16_t olh_epoch,
std::string* petag,
const DoutPrefixProvider* dpp,
optional_yield y)
{
return 0;
}
/* RGWObjectCtx will be moved out of sal */
/* XXX: Placeholder. Should not be needed later after Dan's patch */
void DBObject::set_atomic(RGWObjectCtx* rctx) const
@ -599,7 +674,7 @@ namespace rgw::sal {
return true;
}
int DBObject::get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx)
int DBObject::dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx)
{
return 0;
}
@ -947,13 +1022,6 @@ namespace rgw::sal {
return 0;
}
std::unique_ptr<MultipartUpload>
DBStore::get_multipart_upload(Bucket* bucket, const std::string& oid,
std::optional<std::string> upload_id,
ACLOwner owner, ceph::real_time mtime) {
return nullptr;
}
std::unique_ptr<Writer> DBStore::get_append_writer(const DoutPrefixProvider *dpp,
optional_yield y,
std::unique_ptr<rgw::sal::Object> _head_obj,
@ -1048,7 +1116,7 @@ namespace rgw::sal {
Bucket* bp;
bp = new DBBucket(this, b, u);
ret = bp->get_bucket_info(dpp, y);
ret = bp->load_bucket(dpp, y);
if (ret < 0) {
delete bp;
return ret;
@ -1079,105 +1147,6 @@ namespace rgw::sal {
return get_bucket(dpp, u, b, bucket, y);
}
int DBStore::create_bucket(const DoutPrefixProvider *dpp,
User* u, const rgw_bucket& b,
const string& zonegroup_id,
rgw_placement_rule& placement_rule,
string& swift_ver_location,
const RGWQuotaInfo * pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool *existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket_out,
optional_yield y)
{
int ret;
bufferlist in_data;
RGWBucketInfo master_info;
rgw_bucket *pmaster_bucket = nullptr;
uint32_t *pmaster_num_shards = nullptr;
real_time creation_time;
std::unique_ptr<Bucket> bucket;
obj_version objv, *pobjv = NULL;
/* If it exists, look it up; otherwise create it */
ret = get_bucket(dpp, u, b, &bucket, y);
if (ret < 0 && ret != -ENOENT)
return ret;
if (ret != -ENOENT) {
RGWAccessControlPolicy old_policy(ctx());
*existed = true;
if (swift_ver_location.empty()) {
swift_ver_location = bucket->get_info().swift_ver_location;
}
placement_rule.inherit_from(bucket->get_info().placement_rule);
// don't allow changes to the acl policy
/* int r = rgw_op_get_bucket_policy_from_attr(dpp, this, u, bucket->get_attrs(),
&old_policy, y);
if (r >= 0 && old_policy != policy) {
bucket_out->swap(bucket);
return -EEXIST;
}*/
} else {
bucket = std::make_unique<DBBucket>(this, b, u);
*existed = false;
bucket->set_attrs(attrs);
// XXX: For now single default zone and STANDARD storage class
// supported.
placement_rule.name = "default";
placement_rule.storage_class = "STANDARD";
}
/*
* XXX: If not master zone, fwd the request to master zone.
* For now DBStore has single zone.
*/
std::string zid = zonegroup_id;
/* if (zid.empty()) {
zid = svc()->zone->get_zonegroup().get_id();
} */
if (*existed) {
rgw_placement_rule selected_placement_rule;
/* XXX: Handle this when zone is implemented
ret = svc()->zone->select_bucket_placement(u.get_info(),
zid, placement_rule,
&selected_placement_rule, nullptr, y);
if (selected_placement_rule != info.placement_rule) {
ret = -EEXIST;
bucket_out->swap(bucket);
return ret;
} */
} else {
/* XXX: We may not need to send all these params. Cleanup the unused ones */
ret = getDB()->create_bucket(dpp, u->get_info(), bucket->get_key(),
zid, placement_rule, swift_ver_location, pquota_info,
attrs, info, pobjv, &ep_objv, creation_time,
pmaster_bucket, pmaster_num_shards, y, exclusive);
if (ret == -EEXIST) {
*existed = true;
ret = 0;
} else if (ret != 0) {
return ret;
}
}
bucket->set_version(ep_objv);
bucket->get_info() = info;
bucket_out->swap(bucket);
return ret;
}
bool DBStore::is_meta_master()
{
return true;
@ -1191,11 +1160,6 @@ namespace rgw::sal {
return 0;
}
int DBStore::defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, Bucket* bucket, Object* obj, optional_yield y)
{
return 0;
}
std::string DBStore::zone_unique_id(uint64_t unique_num)
{
return "";

53
src/rgw/rgw_sal_dbstore.h
View File

@ -56,7 +56,22 @@ protected:
}
int list_buckets(const DoutPrefixProvider *dpp, const std::string& marker, const std::string& end_marker,
uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) override;
virtual Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) override;
virtual int create_bucket(const DoutPrefixProvider* dpp,
const rgw_bucket& b,
const std::string& zonegroup_id,
rgw_placement_rule& placement_rule,
std::string& swift_ver_location,
const RGWQuotaInfo* pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool* existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket,
optional_yield y) override;
virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int read_stats(const DoutPrefixProvider *dpp,
optional_yield y, RGWStorageStats* stats,
@ -141,20 +156,18 @@ protected:
DoutPrefixProvider *dpp) override;
virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
virtual int set_acl(const DoutPrefixProvider *dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
virtual int get_bucket_info(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id,
virtual int load_bucket(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int read_stats(const DoutPrefixProvider *dpp, int shard_id,
std::string *bucket_ver, std::string *master_ver,
std::map<RGWObjCategory, RGWStorageStats>& stats,
std::string *max_marker = nullptr,
bool *syncstopped = nullptr) override;
virtual int get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int read_bucket_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int update_container_stats(const DoutPrefixProvider *dpp) override;
virtual int check_bucket_shards(const DoutPrefixProvider *dpp) override;
virtual int chown(const DoutPrefixProvider *dpp, User* new_user, User* old_user, optional_yield y, const std::string* marker = nullptr) override;
virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime) override;
virtual int remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) override;
virtual bool is_owner(User* user) override;
virtual int check_empty(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override;
@ -172,6 +185,9 @@ protected:
virtual std::unique_ptr<Bucket> clone() override {
return std::make_unique<DBBucket>(*this);
}
virtual std::unique_ptr<MultipartUpload> get_multipart_upload(
const std::string& oid, std::optional<std::string> upload_id,
ACLOwner owner={}, ceph::real_time mtime=ceph::real_clock::now()) override;
virtual int list_multiparts(const DoutPrefixProvider *dpp,
const string& prefix,
string& marker,
@ -344,7 +360,6 @@ protected:
virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override;
virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override;
virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) override;
virtual int copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, Object* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider* dpp, optional_yield y) override;
virtual bool is_expired() override;
virtual void gen_rand_obj_instance_name() override;
virtual std::unique_ptr<Object> clone() override {
@ -359,7 +374,7 @@ protected:
const DoutPrefixProvider* dpp,
optional_yield y) override;
virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
virtual int get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override;
virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override;
/* Swift versioning */
virtual int swift_versioning_restore(RGWObjectCtx* obj_ctx,
@ -459,28 +474,10 @@ protected:
virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override;
virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
virtual int create_bucket(const DoutPrefixProvider* dpp,
User* u, const rgw_bucket& b,
const std::string& zonegroup_id,
rgw_placement_rule& placement_rule,
std::string& swift_ver_location,
const RGWQuotaInfo* pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool* existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket,
optional_yield y) override;
virtual bool is_meta_master() override;
virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
bufferlist& in_data, JSONParser *jp, req_info& info,
optional_yield y) override;
virtual int defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, Bucket* bucket, Object* obj,
optional_yield y) override;
virtual Zone* get_zone() { return &zone; }
virtual std::string zone_unique_id(uint64_t unique_num) override;
virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
@ -540,10 +537,6 @@ protected:
virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
const std::string& tenant,
vector<std::unique_ptr<RGWOIDCProvider>>& providers) override;
virtual std::unique_ptr<MultipartUpload>
get_multipart_upload(Bucket* bucket, const std::string& oid,
std::optional<std::string> upload_id=std::nullopt,
ACLOwner owner={}, ceph::real_time mtime=ceph::real_clock::now()) override;
virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp,
optional_yield y,
std::unique_ptr<rgw::sal::Object> _head_obj,

378
src/rgw/rgw_sal_rados.cc
View File

@ -156,10 +156,143 @@ int RadosUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& ma
return 0;
}
Bucket* RadosUser::create_bucket(rgw_bucket& bucket,
ceph::real_time creation_time)
int RadosUser::create_bucket(const DoutPrefixProvider* dpp,
const rgw_bucket& b,
const std::string& zonegroup_id,
rgw_placement_rule& placement_rule,
std::string& swift_ver_location,
const RGWQuotaInfo * pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool* existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket_out,
optional_yield y)
{
return NULL;
int ret;
bufferlist in_data;
RGWBucketInfo master_info;
rgw_bucket* pmaster_bucket;
uint32_t* pmaster_num_shards;
real_time creation_time;
std::unique_ptr<Bucket> bucket;
obj_version objv,* pobjv = NULL;
/* If it exists, look it up; otherwise create it */
ret = store->get_bucket(dpp, this, b, &bucket, y);
if (ret < 0 && ret != -ENOENT)
return ret;
if (ret != -ENOENT) {
RGWAccessControlPolicy old_policy(store->ctx());
*existed = true;
if (swift_ver_location.empty()) {
swift_ver_location = bucket->get_info().swift_ver_location;
}
placement_rule.inherit_from(bucket->get_info().placement_rule);
// don't allow changes to the acl policy
int r = rgw_op_get_bucket_policy_from_attr(dpp, store, this, bucket->get_attrs(),
&old_policy, y);
if (r >= 0 && old_policy != policy) {
bucket_out->swap(bucket);
return -EEXIST;
}
} else {
bucket = std::unique_ptr<Bucket>(new RadosBucket(store, b, this));
*existed = false;
bucket->set_attrs(attrs);
}
if (!store->svc()->zone->is_meta_master()) {
JSONParser jp;
ret = store->forward_request_to_master(dpp, this, NULL, in_data, &jp, req_info, y);
if (ret < 0) {
return ret;
}
JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp);
JSONDecoder::decode_json("object_ver", objv, &jp);
JSONDecoder::decode_json("bucket_info", master_info, &jp);
ldpp_dout(dpp, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl;
std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time);
ldpp_dout(dpp, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl;
pmaster_bucket= &master_info.bucket;
creation_time = master_info.creation_time;
pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards;
pobjv = &objv;
if (master_info.obj_lock_enabled()) {
info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
}
} else {
pmaster_bucket = NULL;
pmaster_num_shards = NULL;
if (obj_lock_enabled)
info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
}
std::string zid = zonegroup_id;
if (zid.empty()) {
zid = store->svc()->zone->get_zonegroup().get_id();
}
if (*existed) {
rgw_placement_rule selected_placement_rule;
ret = store->svc()->zone->select_bucket_placement(dpp, this->get_info(),
zid, placement_rule,
&selected_placement_rule, nullptr, y);
if (selected_placement_rule != info.placement_rule) {
ret = -EEXIST;
bucket_out->swap(bucket);
return ret;
}
} else {
ret = store->getRados()->create_bucket(this->get_info(), bucket->get_key(),
zid, placement_rule, swift_ver_location, pquota_info,
attrs, info, pobjv, &ep_objv, creation_time,
pmaster_bucket, pmaster_num_shards, y, dpp,
exclusive);
if (ret == -EEXIST) {
*existed = true;
/* bucket already existed, might have raced with another bucket creation,
* or might be partial bucket creation that never completed. Read existing
* bucket info, verify that the reported bucket owner is the current user.
* If all is ok then update the user's list of buckets. Otherwise inform
* client about a name conflict.
*/
if (info.owner.compare(this->get_id()) != 0) {
return -EEXIST;
}
ret = 0;
} else if (ret != 0) {
return ret;
}
}
bucket->set_version(ep_objv);
bucket->get_info() = info;
RadosBucket* rbucket = static_cast<RadosBucket*>(bucket.get());
ret = rbucket->link(dpp, this, y, false);
if (ret && !*existed && ret != -EEXIST) {
/* if it exists (or previously existed), don't remove it! */
ret = rbucket->unlink(dpp, this, y);
if (ret < 0) {
ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << ret
<< dendl;
}
} else if (ret == -EEXIST || (ret == 0 && *existed)) {
ret = -ERR_BUCKET_EXISTS;
}
bucket_out->swap(bucket);
return ret;
}
int RadosUser::read_attrs(const DoutPrefixProvider* dpp, optional_yield y)
@ -235,7 +368,7 @@ int RadosBucket::remove_bucket(const DoutPrefixProvider* dpp,
int ret;
// Refresh info
ret = get_bucket_info(dpp, y);
ret = load_bucket(dpp, y);
if (ret < 0)
return ret;
@ -335,11 +468,11 @@ int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool
string bucket_ver, master_ver;
ret = get_bucket_info(dpp, null_yield);
ret = load_bucket(dpp, null_yield);
if (ret < 0)
return ret;
ret = get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
ret = read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
if (ret < 0)
return ret;
@ -455,7 +588,7 @@ int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool
return ret;
}
int RadosBucket::get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y)
int RadosBucket::load_bucket(const DoutPrefixProvider* dpp, optional_yield y)
{
auto obj_ctx = store->svc()->sysobj->init_obj_ctx();
int ret;
@ -476,15 +609,18 @@ int RadosBucket::get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y
.set_attrs(&attrs)
.set_bectx_params(bectx_params));
}
if (ret == 0) {
bucket_version = ep_ot.read_version;
ent.placement_rule = info.placement_rule;
ent.bucket = info.bucket; // we looked up bucket_id
if (ret != 0) {
return ret;
}
bucket_version = ep_ot.read_version;
ret = store->ctl()->bucket->read_bucket_stats(info.bucket, &ent, y, dpp);
return ret;
}
int RadosBucket::get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id,
int RadosBucket::read_stats(const DoutPrefixProvider *dpp, int shard_id,
std::string* bucket_ver, std::string* master_ver,
std::map<RGWObjCategory, RGWStorageStats>& stats,
std::string* max_marker, bool* syncstopped)
@ -492,18 +628,11 @@ int RadosBucket::get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id,
return store->getRados()->get_bucket_stats(dpp, info, shard_id, bucket_ver, master_ver, stats, max_marker, syncstopped);
}
int RadosBucket::get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx)
int RadosBucket::read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx)
{
return store->getRados()->get_bucket_stats_async(dpp, get_info(), shard_id, ctx);
}
int RadosBucket::read_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y)
{
int ret = store->ctl()->bucket->read_bucket_stats(info.bucket, &ent, y, dpp);
info.placement_rule = ent.placement_rule;
return ret;
}
int RadosBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y)
{
return store->ctl()->bucket->sync_user_stats(dpp, owner->get_id(), info, y);
@ -594,19 +723,6 @@ int RadosBucket::put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::r
return store->getRados()->put_bucket_instance_info(info, exclusive, mtime, &attrs, dpp);
}
int RadosBucket::remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y)
{
int r = store->ctl()->bucket->remove_bucket_entrypoint_info(get_key(), y, dpp,
RGWBucketCtl::Bucket::RemoveParams()
.set_objv_tracker(objv));
if (r < 0)
return r;
return store->ctl()->bucket->remove_bucket_instance_info(get_key(), info, y, dpp,
RGWBucketCtl::BucketInstance::RemoveParams()
.set_objv_tracker(objv));
}
/* Make sure to call get_bucket_info() if you need it first */
bool RadosBucket::is_owner(User* user)
{
@ -750,6 +866,15 @@ int RadosBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int max
return ret;
}
std::unique_ptr<MultipartUpload> RadosBucket::get_multipart_upload(
const std::string& oid,
std::optional<std::string> upload_id,
ACLOwner owner, ceph::real_time mtime)
{
return std::make_unique<RadosMultipartUpload>(this->store, this, oid, upload_id,
std::move(owner), mtime);
}
int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp,
const string& prefix,
string& marker,
@ -779,8 +904,8 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp,
rgw_obj_key key(dentry.key);
ACLOwner owner(rgw_user(dentry.meta.owner));
owner.set_name(dentry.meta.owner_display_name);
uploads.push_back(store->get_multipart_upload(
this, key.name, std::nullopt, std::move(owner)));
uploads.push_back(this->get_multipart_upload(key.name,
std::nullopt, std::move(owner)));
}
}
if (common_prefixes) {
@ -925,7 +1050,7 @@ int RadosStore::get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_buc
Bucket* bp;
bp = new RadosBucket(this, b, u);
ret = bp->get_bucket_info(dpp, y);
ret = bp->load_bucket(dpp, y);
if (ret < 0) {
delete bp;
return ret;
@ -956,145 +1081,6 @@ int RadosStore::get_bucket(const DoutPrefixProvider* dpp, User* u, const std::st
return get_bucket(dpp, u, b, bucket, y);
}
int RadosStore::create_bucket(const DoutPrefixProvider* dpp,
User* u, const rgw_bucket& b,
const std::string& zonegroup_id,
rgw_placement_rule& placement_rule,
std::string& swift_ver_location,
const RGWQuotaInfo * pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool* existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket_out,
optional_yield y)
{
int ret;
bufferlist in_data;
RGWBucketInfo master_info;
rgw_bucket* pmaster_bucket;
uint32_t* pmaster_num_shards;
real_time creation_time;
std::unique_ptr<Bucket> bucket;
obj_version objv,* pobjv = NULL;
/* If it exists, look it up; otherwise create it */
ret = get_bucket(dpp, u, b, &bucket, y);
if (ret < 0 && ret != -ENOENT)
return ret;
if (ret != -ENOENT) {
RGWAccessControlPolicy old_policy(ctx());
*existed = true;
if (swift_ver_location.empty()) {
swift_ver_location = bucket->get_info().swift_ver_location;
}
placement_rule.inherit_from(bucket->get_info().placement_rule);
// don't allow changes to the acl policy
int r = rgw_op_get_bucket_policy_from_attr(dpp, this, u, bucket->get_attrs(),
&old_policy, y);
if (r >= 0 && old_policy != policy) {
bucket_out->swap(bucket);
return -EEXIST;
}
} else {
bucket = std::unique_ptr<Bucket>(new RadosBucket(this, b, u));
*existed = false;
bucket->set_attrs(attrs);
}
if (!svc()->zone->is_meta_master()) {
JSONParser jp;
ret = forward_request_to_master(dpp, u, NULL, in_data, &jp, req_info, y);
if (ret < 0) {
return ret;
}
JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp);
JSONDecoder::decode_json("object_ver", objv, &jp);
JSONDecoder::decode_json("bucket_info", master_info, &jp);
ldpp_dout(dpp, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl;
std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time);
ldpp_dout(dpp, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl;
pmaster_bucket= &master_info.bucket;
creation_time = master_info.creation_time;
pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards;
pobjv = &objv;
if (master_info.obj_lock_enabled()) {
info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
}
} else {
pmaster_bucket = NULL;
pmaster_num_shards = NULL;
if (obj_lock_enabled)
info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED;
}
std::string zid = zonegroup_id;
if (zid.empty()) {
zid = svc()->zone->get_zonegroup().get_id();
}
if (*existed) {
rgw_placement_rule selected_placement_rule;
ret = svc()->zone->select_bucket_placement(dpp, u->get_info(),
zid, placement_rule,
&selected_placement_rule, nullptr, y);
if (selected_placement_rule != info.placement_rule) {
ret = -EEXIST;
bucket_out->swap(bucket);
return ret;
}
} else {
ret = getRados()->create_bucket(u->get_info(), bucket->get_key(),
zid, placement_rule, swift_ver_location, pquota_info,
attrs, info, pobjv, &ep_objv, creation_time,
pmaster_bucket, pmaster_num_shards, y, dpp,
exclusive);
if (ret == -EEXIST) {
*existed = true;
/* bucket already existed, might have raced with another bucket creation,
* or might be partial bucket creation that never completed. Read existing
* bucket info, verify that the reported bucket owner is the current user.
* If all is ok then update the user's list of buckets. Otherwise inform
* client about a name conflict.
*/
if (info.owner.compare(u->get_id()) != 0) {
return -EEXIST;
}
ret = 0;
} else if (ret != 0) {
return ret;
}
}
bucket->set_version(ep_objv);
bucket->get_info() = info;
RadosBucket* rbucket = static_cast<RadosBucket*>(bucket.get());
ret = rbucket->link(dpp, u, y, false);
if (ret && !*existed && ret != -EEXIST) {
/* if it exists (or previously existed), don't remove it! */
ret = rbucket->unlink(dpp, u, y);
if (ret < 0) {
ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << ret
<< dendl;
}
} else if (ret == -EEXIST || (ret == 0 && *existed)) {
ret = -ERR_BUCKET_EXISTS;
}
bucket_out->swap(bucket);
return ret;
}
bool RadosStore::is_meta_master()
{
return svc()->zone->is_meta_master();
@ -1133,11 +1119,6 @@ int RadosStore::forward_request_to_master(const DoutPrefixProvider *dpp, User* u
return 0;
}
int RadosStore::defer_gc(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Bucket* bucket, Object* obj, optional_yield y)
{
return rados->defer_gc(dpp, rctx, bucket->get_info(), obj->get_obj(), y);
}
std::string RadosStore::zone_unique_id(uint64_t unique_num)
{
return svc()->zone_utils->unique_id(unique_num);
@ -1444,15 +1425,6 @@ int RadosStore::get_oidc_providers(const DoutPrefixProvider *dpp,
return 0;
}
std::unique_ptr<MultipartUpload>
RadosStore::get_multipart_upload(Bucket* bucket, const std::string& oid,
std::optional<std::string> upload_id,
ACLOwner owner, ceph::real_time mtime)
{
return std::make_unique<RadosMultipartUpload>(this, bucket, oid, upload_id,
std::move(owner), mtime);
}
std::unique_ptr<Writer> RadosStore::get_append_writer(const DoutPrefixProvider *dpp,
optional_yield y,
std::unique_ptr<rgw::sal::Object> _head_obj,
@ -1555,32 +1527,6 @@ int RadosObject::delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* r
return set_obj_attrs(dpp, rctx, nullptr, &rmattr, y);
}
int RadosObject::copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket,
Object* dest_obj,
uint16_t olh_epoch,
std::string* petag,
const DoutPrefixProvider* dpp,
optional_yield y)
{
Attrs attrset;
RGWRados::Object op_target(store->getRados(), dest_bucket->get_info(), rctx, get_obj());
RGWRados::Object::Read read_op(&op_target);
int ret = read_attrs(dpp, read_op, y);
if (ret < 0)
return ret;
attrset = attrs;
attrset.erase(RGW_ATTR_ID_TAG);
attrset.erase(RGW_ATTR_TAIL_TAG);
return store->getRados()->copy_obj_data(rctx, dest_bucket,
dest_bucket->get_info().placement_rule, read_op,
obj_size - 1, dest_obj, NULL, mtime, attrset, 0,
real_time(), NULL, dpp, y);
}
void RadosObject::set_compressed(RGWObjectCtx* rctx) {
rgw_obj obj = get_obj();
store->getRados()->set_compressed(rctx, obj);
@ -1737,7 +1683,7 @@ bool RadosObject::placement_rules_match(rgw_placement_rule& r1, rgw_placement_ru
return p1 == p2;
}
int RadosObject::get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx)
int RadosObject::dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx)
{
int ret;
RGWObjManifest *manifest{nullptr};

58
src/rgw/rgw_sal_rados.h
View File

@ -55,7 +55,22 @@ class RadosUser : public User {
int list_buckets(const DoutPrefixProvider* dpp, const std::string& marker, const std::string& end_marker,
uint64_t max, bool need_stats, BucketList& buckets,
optional_yield y) override;
virtual Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) override;
virtual int create_bucket(const DoutPrefixProvider* dpp,
const rgw_bucket& b,
const std::string& zonegroup_id,
rgw_placement_rule& placement_rule,
std::string& swift_ver_location,
const RGWQuotaInfo * pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool* existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket,
optional_yield y) override;
virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int read_stats(const DoutPrefixProvider *dpp,
optional_yield y, RGWStorageStats* stats,
@ -157,7 +172,6 @@ class RadosObject : public Object {
virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override;
virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override;
virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) override;
virtual int copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, Object* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider* dpp, optional_yield y) override;
virtual bool is_expired() override;
virtual void gen_rand_obj_instance_name() override;
void get_raw_obj(rgw_raw_obj* raw_obj);
@ -173,7 +187,7 @@ class RadosObject : public Object {
const DoutPrefixProvider* dpp,
optional_yield y) override;
virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override;
virtual int get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override;
virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override;
/* Swift versioning */
virtual int swift_versioning_restore(RGWObjectCtx* obj_ctx,
@ -275,20 +289,18 @@ class RadosBucket : public Bucket {
DoutPrefixProvider *dpp) override;
virtual RGWAccessControlPolicy& get_acl(void) override { return acls; }
virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) override;
virtual int get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id,
virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int read_stats(const DoutPrefixProvider *dpp, int shard_id,
std::string* bucket_ver, std::string* master_ver,
std::map<RGWObjCategory, RGWStorageStats>& stats,
std::string* max_marker = nullptr,
bool* syncstopped = nullptr) override;
virtual int get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int read_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int update_container_stats(const DoutPrefixProvider* dpp) override;
virtual int check_bucket_shards(const DoutPrefixProvider* dpp) override;
virtual int chown(const DoutPrefixProvider* dpp, User* new_user, User* old_user, optional_yield y, const std::string* marker = nullptr) override;
virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime) override;
virtual int remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) override;
virtual bool is_owner(User* user) override;
virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override;
@ -306,6 +318,10 @@ class RadosBucket : public Bucket {
virtual std::unique_ptr<Bucket> clone() override {
return std::make_unique<RadosBucket>(*this);
}
virtual std::unique_ptr<MultipartUpload> get_multipart_upload(
const std::string& oid,
std::optional<std::string> upload_id=std::nullopt,
ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override;
virtual int list_multiparts(const DoutPrefixProvider *dpp,
const std::string& prefix,
std::string& marker,
@ -321,7 +337,7 @@ class RadosBucket : public Bucket {
private:
int link(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true, RGWObjVersionTracker* objv = nullptr);
int unlink(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true);
friend class RadosStore;
friend class RadosUser;
};
class RadosZone : public Zone {
@ -366,28 +382,10 @@ class RadosStore : public Store {
virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr<Bucket>* bucket) override;
virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr<Bucket>* bucket, optional_yield y) override;
virtual int create_bucket(const DoutPrefixProvider* dpp,
User* u, const rgw_bucket& b,
const std::string& zonegroup_id,
rgw_placement_rule& placement_rule,
std::string& swift_ver_location,
const RGWQuotaInfo * pquota_info,
const RGWAccessControlPolicy& policy,
Attrs& attrs,
RGWBucketInfo& info,
obj_version& ep_objv,
bool exclusive,
bool obj_lock_enabled,
bool* existed,
req_info& req_info,
std::unique_ptr<Bucket>* bucket,
optional_yield y) override;
virtual bool is_meta_master() override;
virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv,
bufferlist& in_data, JSONParser* jp, req_info& info,
optional_yield y) override;
virtual int defer_gc(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Bucket* bucket, Object* obj,
optional_yield y) override;
virtual Zone* get_zone() { return &zone; }
virtual std::string zone_unique_id(uint64_t unique_num) override;
virtual std::string zone_unique_trans_id(const uint64_t unique_num) override;
@ -444,10 +442,6 @@ class RadosStore : public Store {
virtual int get_oidc_providers(const DoutPrefixProvider *dpp,
const std::string& tenant,
std::vector<std::unique_ptr<RGWOIDCProvider>>& providers) override;
virtual std::unique_ptr<MultipartUpload>
get_multipart_upload(Bucket* bucket, const std::string& oid,
std::optional<std::string> upload_id=std::nullopt,
ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override;
virtual std::unique_ptr<Writer> get_append_writer(const DoutPrefixProvider *dpp,
optional_yield y,
std::unique_ptr<rgw::sal::Object> _head_obj,
@ -534,7 +528,7 @@ public:
virtual const std::string& get_key() const override { return mp_obj.get_key(); }
virtual const std::string& get_upload_id() const override { return mp_obj.get_upload_id(); }
virtual const ACLOwner& get_owner() const override { return owner; }
virtual ceph::real_time get_mtime() const override { return mtime; }
virtual ceph::real_time& get_mtime() override { return mtime; }
virtual std::unique_ptr<rgw::sal::Object> get_meta_obj() override;
virtual int init(const DoutPrefixProvider* dpp, optional_yield y, RGWObjectCtx* obj_ctx, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) override;
virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct,

15
src/rgw/rgw_user.cc
View File

@ -68,7 +68,7 @@ int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* stor
auto& bucket = i->second;
ret = bucket->get_bucket_info(dpp, y);
ret = bucket->load_bucket(dpp, y);
if (ret < 0) {
ldpp_dout(dpp, 0) << "ERROR: could not read bucket info: bucket=" << bucket << " ret=" << ret << dendl;
continue;
@ -97,7 +97,7 @@ int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* stor
int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp,
rgw::sal::Store* store,
rgw::sal::User* user,
map<string, cls_user_bucket_entry>& buckets_usage_map,
map<string, bucket_meta_entry>& buckets_usage_map,
optional_yield y)
{
CephContext *cct = store->ctx();
@ -118,13 +118,16 @@ int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp,
marker = i.first;
auto& bucket_ent = i.second;
ret = bucket_ent->read_bucket_stats(dpp, y);
ret = bucket_ent->load_bucket(dpp, y);
if (ret < 0) {
ldpp_dout(dpp, 0) << "ERROR: could not get bucket stats: ret=" << ret << dendl;
return ret;
}
cls_user_bucket_entry entry;
bucket_ent->convert(&entry);
bucket_meta_entry entry;
entry.size = bucket_ent->get_size();
entry.size_rounded = bucket_ent->get_size_rounded();
entry.creation_time = bucket_ent->get_creation_time();
entry.count = bucket_ent->get_count();
buckets_usage_map.emplace(bucket_ent->get_name(), entry);
}
done = (buckets.count() < max_entries);
@ -1733,7 +1736,7 @@ int RGWUser::execute_rename(const DoutPrefixProvider *dpp, RGWUserAdminOpState&
auto& bucket = it->second;
marker = it->first;
ret = bucket->get_bucket_info(dpp, y);
ret = bucket->load_bucket(dpp, y);
if (ret < 0) {
set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name());
return ret;

10
src/rgw/rgw_user.h
View File

@ -57,10 +57,18 @@ struct RGWUID
};
WRITE_CLASS_ENCODER(RGWUID)
/** Entry for bucket metadata collection */
struct bucket_meta_entry {
size_t size;
size_t size_rounded;
ceph::real_time creation_time;
uint64_t count;
};
extern int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::User* user, optional_yield y);
extern int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp,
rgw::sal::Store* store, rgw::sal::User* user,
std::map<std::string, cls_user_bucket_entry>& buckets_usage_map, optional_yield y);
std::map<std::string, bucket_meta_entry>& buckets_usage_map, optional_yield y);
/**
* Get the anonymous (ie, unauthenticated) user info.

4
src/test/rgw/test_rgw_lua.cc
View File

@ -76,8 +76,8 @@ public:
return 0;
}
virtual sal::Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) override {
return nullptr;
virtual int create_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, const std::string& zonegroup_id, rgw_placement_rule& placement_rule, std::string& swift_ver_location, const RGWQuotaInfo* pquota_info, const RGWAccessControlPolicy& policy, sal::Attrs& attrs, RGWBucketInfo& info, obj_version& ep_objv, bool exclusive, bool obj_lock_enabled, bool* existed, req_info& req_info, std::unique_ptr<sal::Bucket>* bucket, optional_yield y) override {
return 0;
}
virtual int read_attrs(const DoutPrefixProvider *dpp, optional_yield y) override {