RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-01-03 17:42:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

rgw: implicit rgw_bucket placement, manifest uses rgw_raw_obj

Browse Source 
Two main changes here:
1. Newly created rgw_bucket does not have a predetermined placement pools
assigned to it. The placement_id param in the objects themselves points
at where the data is located. This affects object's tail location, head
is located where the bucket instance's placement rule points at.
2. Modify object manifest to use rgw_raw_obj instead of rgw_obj.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
This commit is contained in:
Yehuda Sadeh 2016-10-31 15:56:26 -07:00
parent df534b413c
commit 9c1b828e83
16 changed files with 593 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/cls/user/cls_user_types.cc
View File

@ -11,8 +11,6 @@ void cls_user_gen_test_bucket(cls_user_bucket *bucket, int i)
snprintf(buf, sizeof(buf), ".%d", i);
bucket->name = string("buck") + buf;
bucket->data_pool = string(".data.pool") + buf;
bucket->index_pool = string(".index.pool") + buf;
bucket->marker = string("mark") + buf;
bucket->bucket_id = string("bucket.id") + buf;
}
@ -20,9 +18,6 @@ void cls_user_gen_test_bucket(cls_user_bucket *bucket, int i)
void cls_user_bucket::dump(Formatter *f) const
{
encode_json("name", name, f);
encode_json("data_pool", data_pool,f);
encode_json("data_extra_pool", data_extra_pool,f);
encode_json("index_pool", index_pool,f);
encode_json("marker", marker,f);
encode_json("bucket_id", bucket_id,f);
}

47
src/cls/user/cls_user_types.h
View File

@ -14,26 +14,34 @@
*/
struct cls_user_bucket {
std::string name;
std::string data_pool;
std::string index_pool;
std::string marker;
std::string bucket_id;
std::string data_extra_pool;
std::string placement_id;
struct {
std::string data_pool;
std::string index_pool;
std::string data_extra_pool;
} explicit_placement;
void encode(bufferlist& bl) const {
ENCODE_START(7, 3, bl);
ENCODE_START(8, 8, bl);
::encode(name, bl);
::encode(data_pool, bl);
::encode(marker, bl);
::encode(bucket_id, bl);
::encode(index_pool, bl);
::encode(data_extra_pool, bl);
::encode(placement_id, bl);
if (placement_id.empty()) {
::encode(explicit_placement.data_pool, bl);
::encode(explicit_placement.index_pool, bl);
::encode(explicit_placement.data_extra_pool, bl);
}
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
DECODE_START_LEGACY_COMPAT_LEN(7, 3, 3, bl);
DECODE_START_LEGACY_COMPAT_LEN(8, 3, 3, bl);
::decode(name, bl);
::decode(data_pool, bl);
if (struct_v < 8) {
::decode(explicit_placement.data_pool, bl);
}
if (struct_v >= 2) {
::decode(marker, bl);
if (struct_v <= 3) {
@ -46,13 +54,22 @@ struct cls_user_bucket {
::decode(bucket_id, bl);
}
}
if (struct_v >= 5) {
::decode(index_pool, bl);
if (struct_v < 8) {
if (struct_v >= 5) {
::decode(explicit_placement.index_pool, bl);
} else {
explicit_placement.index_pool = explicit_placement.data_pool;
}
if (struct_v >= 7) {
::decode(explicit_placement.data_extra_pool, bl);
}
} else {
index_pool = data_pool;
}
if (struct_v >= 7) {
::decode(data_extra_pool, bl);
::decode(placement_id, bl);
if (placement_id.empty()) {
::decode(explicit_placement.data_pool, bl);
::decode(explicit_placement.index_pool, bl);
::decode(explicit_placement.data_extra_pool, bl);
}
}
DECODE_FINISH(bl);
}

14
src/rgw/rgw_admin.cc
View File

@ -1016,14 +1016,14 @@ int bucket_stats(rgw_bucket& bucket, int shard_id, Formatter *formatter)
map<RGWObjCategory, RGWStorageStats> stats;
string bucket_ver, master_ver;
string max_marker;
int ret = store->get_bucket_stats(bucket, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
int ret = store->get_bucket_stats(bucket_info, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0) {
cerr << "error getting bucket stats ret=" << ret << std::endl;
return ret;
}
formatter->open_object_section("stats");
formatter->dump_string("bucket", bucket.name);
::encode_json("placement", bucket.placement, formatter);
::encode_json("explicit_placement", bucket.explicit_placement, formatter);
formatter->dump_string("id", bucket.bucket_id);
formatter->dump_string("marker", bucket.marker);
@ -5436,7 +5436,7 @@ next:
formatter->open_array_section("objects");
while (is_truncated) {
map<string, RGWObjEnt> result;
int r = store->cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, 1000, true,
int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true,
result, &is_truncated, &marker,
bucket_object_check_filter);
@ -5659,7 +5659,7 @@ next:
list<rgw_obj_key> oid_list;
rgw_obj_key key(object, object_version);
oid_list.push_back(key);
ret = store->remove_objs_from_index(bucket, oid_list);
ret = store->remove_objs_from_index(bucket_info, oid_list);
if (ret < 0) {
cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
return 1;
@ -6395,7 +6395,7 @@ next:
do {
list<rgw_bi_log_entry> entries;
ret = store->list_bi_log_entries(bucket, shard_id, marker, max_entries - count, entries, &truncated);
ret = store->list_bi_log_entries(bucket_info, shard_id, marker, max_entries - count, entries, &truncated);
if (ret < 0) {
cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;
@ -6504,7 +6504,7 @@ next:
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
}
ret = store->trim_bi_log_entries(bucket, shard_id, start_marker, end_marker);
ret = store->trim_bi_log_entries(bucket_info, shard_id, start_marker, end_marker);
if (ret < 0) {
cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;
@ -6523,7 +6523,7 @@ next:
return -ret;
}
map<int, string> markers;
ret = store->get_bi_log_status(bucket, shard_id, markers);
ret = store->get_bi_log_status(bucket_info, shard_id, markers);
if (ret < 0) {
cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;

66
src/rgw/rgw_bucket.cc
View File

@ -151,13 +151,13 @@ int rgw_read_user_buckets(RGWRados * store,
return 0;
}
int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket)
int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, const RGWBucketInfo& bucket_info)
{
string buckets_obj_id;
rgw_get_buckets_obj(user_id, buckets_obj_id);
rgw_raw_obj obj(store->get_zone_params().user_uid_pool, buckets_obj_id);
return store->cls_user_sync_bucket_stats(obj, bucket);
return store->cls_user_sync_bucket_stats(obj, bucket_info);
}
int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name)
@ -170,7 +170,7 @@ int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const
return ret;
}
ret = rgw_bucket_sync_user_stats(store, bucket_info.owner, bucket_info.bucket);
ret = rgw_bucket_sync_user_stats(store, bucket_info.owner, bucket_info);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: could not sync user stats for bucket " << bucket_name << ": ret=" << ret << dendl;
return ret;
@ -480,7 +480,6 @@ void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_id,
if (actual_bucket.name.compare(bucket.name) != 0 ||
actual_bucket.tenant.compare(bucket.tenant) != 0 ||
actual_bucket.placement.compare(bucket.placement) != 0 ||
actual_bucket.marker.compare(bucket.marker) != 0 ||
actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
@ -529,11 +528,11 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
string bucket_ver, master_ver;
ret = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
if (ret < 0)
return ret;
ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
ret = store->get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
if (ret < 0)
return ret;
@ -564,14 +563,14 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
}
ret = rgw_bucket_sync_user_stats(store, bucket.tenant, bucket.name);
ret = rgw_bucket_sync_user_stats(store, bucket.tenant, info);
if ( ret < 0) {
dout(1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl;
}
RGWObjVersionTracker objv_tracker;
ret = store->delete_bucket(bucket, objv_tracker);
ret = store->delete_bucket(info, objv_tracker);
if (ret < 0) {
lderr(store->ctx()) << "ERROR: could not remove bucket " << bucket.name << dendl;
return ret;
@ -620,11 +619,11 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket,
string bucket_ver, master_ver;
ret = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
if (ret < 0)
return ret;
ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name, info, NULL);
ret = store->get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL);
if (ret < 0)
return ret;
@ -664,7 +663,7 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket,
RGWObjManifest::obj_iterator miter = manifest.obj_begin();
rgw_obj head_obj = manifest.get_obj();
rgw_raw_obj raw_head_obj;
RGWRados::obj_to_raw(head_obj, &raw_head_obj);
store->obj_to_raw(head_obj, &raw_head_obj);
for (; miter != manifest.obj_end() && max_aio--; ++miter) {
@ -677,7 +676,7 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket,
max_aio = concurrent_max;
}
rgw_raw_obj last_obj = miter.get_location();
rgw_raw_obj last_obj = miter.get_location().get_raw_obj(store);
if (last_obj == raw_head_obj) {
// have the head obj deleted at the end
continue;
@ -690,7 +689,7 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket,
}
} // for all shadow objs
ret = store->delete_obj_aio(head_obj, bucket, info, astate, handles, keep_index_consistent);
ret = store->delete_obj_aio(head_obj, info, astate, handles, keep_index_consistent);
if (ret < 0) {
lderr(store->ctx()) << "ERROR: delete obj aio failed with " << ret << dendl;
return ret;
@ -719,7 +718,7 @@ int rgw_remove_bucket_bypass_gc(RGWRados *store, rgw_bucket& bucket,
return ret;
}
ret = rgw_bucket_sync_user_stats(store, bucket.tenant, bucket.name);
ret = rgw_bucket_sync_user_stats(store, bucket.tenant, info);
if (ret < 0) {
dout(1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl;
}
@ -1073,7 +1072,7 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state,
return 0;
if (fix_index) {
int r = store->remove_objs_from_index(bucket, objs_to_unlink);
int r = store->remove_objs_from_index(bucket_info, objs_to_unlink);
if (r < 0) {
set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " +
cpp_strerror(-r));
@ -1091,6 +1090,7 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
{
bool fix_index = op_state.will_fix_index();
rgw_bucket bucket = op_state.get_bucket();
if (!fix_index) {
@ -1098,11 +1098,7 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
return -EINVAL;
}
/*
dout(0) << "Checking objects, decreasing bucket 2-phase commit timeout.\n"\
<< "** Note that timeout will reset only when operation completes successfully **" << dendl;
*/
store->cls_obj_set_bucket_tag_timeout(bucket, BUCKET_TAG_TIMEOUT);
store->cls_obj_set_bucket_tag_timeout(bucket_info, BUCKET_TAG_TIMEOUT);
string prefix;
rgw_obj_key marker;
@ -1113,7 +1109,7 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
while (is_truncated) {
map<string, RGWObjEnt> result;
int r = store->cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, 1000, true,
int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true,
result, &is_truncated, &marker,
bucket_object_check_filter);
if (r == -ENOENT) {
@ -1130,7 +1126,7 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
formatter->close_section();
store->cls_obj_set_bucket_tag_timeout(bucket, 0);
store->cls_obj_set_bucket_tag_timeout(bucket_info, 0);
return 0;
}
@ -1144,14 +1140,14 @@ int RGWBucket::check_index(RGWBucketAdminOpState& op_state,
rgw_bucket bucket = op_state.get_bucket();
bool fix_index = op_state.will_fix_index();
int r = store->bucket_check_index(bucket, &existing_stats, &calculated_stats);
int r = store->bucket_check_index(bucket_info, &existing_stats, &calculated_stats);
if (r < 0) {
set_err_msg(err_msg, "failed to check index error=" + cpp_strerror(-r));
return r;
}
if (fix_index) {
r = store->bucket_rebuild_index(bucket);
r = store->bucket_rebuild_index(bucket_info);
if (r < 0) {
set_err_msg(err_msg, "failed to rebuild index err=" + cpp_strerror(-r));
return r;
@ -1367,7 +1363,6 @@ int RGWBucketAdminOp::remove_object(RGWRados *store, RGWBucketAdminOpState& op_s
static int bucket_stats(RGWRados *store, const std::string& tenant_name, std::string& bucket_name, Formatter *formatter)
{
RGWBucketInfo bucket_info;
rgw_bucket bucket;
map<RGWObjCategory, RGWStorageStats> stats;
real_time mtime;
@ -1376,11 +1371,11 @@ static int bucket_stats(RGWRados *store, const std::string& tenant_name, std::st
if (r < 0)
return r;
bucket = bucket_info.bucket;
rgw_bucket& bucket = bucket_info.bucket;
string bucket_ver, master_ver;
string max_marker;
int ret = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker);
int ret = store->get_bucket_stats(bucket_info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0) {
cerr << "error getting bucket stats ret=" << ret << std::endl;
return ret;
@ -1390,7 +1385,6 @@ static int bucket_stats(RGWRados *store, const std::string& tenant_name, std::st
formatter->open_object_section("stats");
formatter->dump_string("bucket", bucket.name);
encode_json("placement", bucket.placement, formatter);
formatter->dump_string("id", bucket.bucket_id);
formatter->dump_string("marker", bucket.marker);
::encode_json("owner", bucket_info.owner, formatter);
@ -2122,21 +2116,19 @@ public:
string bucket_name;
parse_bucket(key, tenant_name, bucket_name);
rgw_bucket bucket;
RGWZonePlacementInfo rule_info;
ret = store->set_bucket_location_by_rule(bci.info.placement_rule,
tenant_name, bucket_name, bucket, &rule_info);
bci.info.bucket.name = bucket_name;
bci.info.bucket.tenant = tenant_name;
ret = store->select_bucket_location_by_rule(bci.info.placement_rule, bci.info.bucket, &rule_info);
if (ret < 0) {
ldout(store->ctx(), 0) << "ERROR: select_bucket_placement() returned " << ret << dendl;
return ret;
}
bci.info.bucket.tenant = bucket.tenant;
bci.info.bucket.placement = bucket.placement;
bci.info.index_type = rule_info.index_type;
} else {
/* existing bucket, keep its placement pools */
bci.info.bucket.placement = old_bci.info.bucket.placement;
bci.info.index_type = old_bci.info.index_type;
/* existing bucket, keep its placement */
bci.info.bucket.explicit_placement = old_bci.info.bucket.explicit_placement;
bci.info.placement_rule = old_bci.info.placement_rule;
}
// are we actually going to perform this put, or is it too old?
@ -2157,7 +2149,7 @@ public:
objv_tracker = bci.info.objv_tracker;
ret = store->init_bucket_index(bci.info.bucket, bci.info.num_shards);
ret = store->init_bucket_index(bci.info, bci.info.num_shards);
if (ret < 0)
return ret;

2
src/rgw/rgw_bucket.h
View File

@ -46,7 +46,7 @@ extern int rgw_bucket_delete_bucket_obj(RGWRados *store,
const string& bucket_name,
RGWObjVersionTracker& objv_tracker);
extern int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket);
extern int rgw_bucket_sync_user_stats(RGWRados *store, const rgw_user& user_id, const RGWBucketInfo& bucket_info);
extern int rgw_bucket_sync_user_stats(RGWRados *store, const string& tenant_name, const string& bucket_name);
extern void rgw_make_bucket_entry_name(const string& tenant_name,

3
src/rgw/rgw_common.cc
View File

@ -1440,7 +1440,8 @@ void rgw_raw_obj::decode_from_rgw_obj(bufferlist::iterator& bl)
rgw_obj old_obj;
::decode(old_obj, bl);
RGWRados::obj_to_raw(old_obj, this);
get_obj_bucket_and_oid_loc(old_obj, oid, loc);
pool = old_obj.get_explicit_data_pool();
}
std::string rgw_bucket::get_key(char tenant_delim, char id_delim) const

77
src/rgw/rgw_common.h
View File

@ -912,7 +912,7 @@ struct rgw_bucket {
std::string name;
std::string marker;
std::string bucket_id;
rgw_data_placement_target placement;
rgw_data_placement_target explicit_placement;
std::string oid; /*
* runtime in-memory only info. If not empty, points to the bucket instance object
@ -925,32 +925,40 @@ struct rgw_bucket {
name(b.name),
marker(b.marker),
bucket_id(b.bucket_id),
placement(b.data_pool, b.data_extra_pool, b.index_pool) {}
explicit_placement(b.explicit_placement.data_pool,
b.explicit_placement.data_extra_pool,
b.explicit_placement.index_pool) {}
void convert(cls_user_bucket *b) {
void convert(cls_user_bucket *b) const {
b->name = name;
b->data_pool = placement.data_pool.to_str();
b->data_extra_pool = placement.data_extra_pool.to_str();
b->index_pool = placement.index_pool.to_str();
b->marker = marker;
b->bucket_id = bucket_id;
b->explicit_placement.data_pool = explicit_placement.data_pool.to_str();
b->explicit_placement.data_extra_pool = explicit_placement.data_extra_pool.to_str();
b->explicit_placement.index_pool = explicit_placement.index_pool.to_str();
}
void encode(bufferlist& bl) const {
ENCODE_START(9, 3, bl);
ENCODE_START(10, 10, bl);
::encode(name, bl);
::encode(placement.data_pool.name, bl);
::encode(marker, bl);
::encode(bucket_id, bl);
::encode(placement.index_pool.name, bl);
::encode(placement.data_extra_pool.name, bl);
::encode(tenant, bl);
bool encode_explicit = !explicit_placement.data_pool.empty();
::encode(encode_explicit, bl);
if (encode_explicit) {
::encode(explicit_placement.data_pool, bl);
::encode(explicit_placement.data_extra_pool, bl);
::encode(explicit_placement.index_pool, bl);
}
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
DECODE_START_LEGACY_COMPAT_LEN(9, 3, 3, bl);
DECODE_START_LEGACY_COMPAT_LEN(10, 3, 3, bl);
::decode(name, bl);
::decode(placement.data_pool.name, bl);
if (struct_v < 10) {
::decode(explicit_placement.data_pool.name, bl);
}
if (struct_v >= 2) {
::decode(marker, bl);
if (struct_v <= 3) {
@ -963,17 +971,28 @@ struct rgw_bucket {
::decode(bucket_id, bl);
}
}
if (struct_v >= 5) {
::decode(placement.index_pool.name, bl);
} else {
placement.index_pool = placement.data_pool;
}
if (struct_v >= 7) {
::decode(placement.data_extra_pool.name, bl);
if (struct_v < 10) {
if (struct_v >= 5) {
::decode(explicit_placement.index_pool.name, bl);
} else {
explicit_placement.index_pool = explicit_placement.data_pool;
}
if (struct_v >= 7) {
::decode(explicit_placement.data_extra_pool.name, bl);
}
}
if (struct_v >= 8) {
::decode(tenant, bl);
}
if (struct_v >= 10) {
bool decode_explicit = !explicit_placement.data_pool.empty();
::decode(decode_explicit, bl);
if (decode_explicit) {
::decode(explicit_placement.data_pool, bl);
::decode(explicit_placement.data_extra_pool, bl);
::decode(explicit_placement.index_pool, bl);
}
}
DECODE_FINISH(bl);
}
@ -982,7 +1001,7 @@ struct rgw_bucket {
char id_delim = ':') const;
const rgw_pool& get_data_extra_pool() const {
return placement.get_data_extra_pool();
return explicit_placement.get_data_extra_pool();
}
void dump(Formatter *f) const;
@ -1580,7 +1599,7 @@ struct RGWBucketEnt {
count(e.count) {
}
void convert(cls_user_bucket_entry *b) {
void convert(cls_user_bucket_entry *b) const {
bucket.convert(&b->bucket);
b->size = size;
b->size_rounded = size_rounded;
@ -1642,6 +1661,7 @@ public:
const std::string& get_loc() const { return loc; }
const std::string& get_instance() const { return instance; }
rgw_bucket bucket;
std::string placement_id;
std::string ns;
bool in_extra_data; /* in-memory only member, does not serialize */
@ -1905,14 +1925,6 @@ public:
return in_extra_data;
}
const rgw_pool& get_data_pool() const {
if (!in_extra_data) {
return bucket.placement.data_pool;
} else {
return bucket.placement.data_extra_pool;
}
}
void encode(bufferlist& bl) const {
ENCODE_START(5, 3, bl);
::encode(bucket.name, bl);
@ -1981,6 +1993,13 @@ public:
return (r < 0);
}
const rgw_pool& get_explicit_data_pool() {
if (!in_extra_data || bucket.explicit_placement.data_extra_pool.empty()) {
return bucket.explicit_placement.data_pool;
}
return bucket.explicit_placement.data_extra_pool;
}
};
WRITE_CLASS_ENCODER(rgw_obj)

16
src/rgw/rgw_dencoder.cc
View File

@ -18,8 +18,8 @@ static void init_bucket(rgw_bucket *b, const char *t, const char *n, const char
b->name = n;
b->marker = m;
b->bucket_id = id;
b->placement.data_pool = rgw_pool(dp);
b->placement.index_pool = rgw_pool(ip);
b->explicit_placement.data_pool = rgw_pool(dp);
b->explicit_placement.index_pool = rgw_pool(ip);
}
void RGWObjManifestPart::generate_test_instances(std::list<RGWObjManifestPart*>& o)
@ -112,14 +112,12 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o)
void RGWObjManifest::obj_iterator::update_location()
{
if (manifest->explicit_objs) {
RGWRados::obj_to_raw(explicit_iter->second.loc, &location);
location = explicit_iter->second.loc;
return;
}
const rgw_raw_obj& head = manifest->get_head();
if (ofs < manifest->get_head_size()) {
location = head;
location = manifest->get_obj();
return;
}
@ -159,7 +157,7 @@ void RGWObjManifest::generate_test_instances(std::list<RGWObjManifest*>& o)
o.push_back(new RGWObjManifest);
}
void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_raw_obj *location)
void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location)
{
string oid;
if (!override_prefix || override_prefix->empty()) {
@ -171,7 +169,7 @@ void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_st
if (!cur_part_id) {
if (ofs < max_head_size) {
*location = head_obj;
*location = obj;
return;
} else {
char buf[16];
@ -208,7 +206,7 @@ void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_st
// to get the right shadow object location
loc.set_instance(tail_instance);
RGWRados::obj_to_raw(loc, location);
*location = loc;
}

13
src/rgw/rgw_json_enc.cc
View File

@ -94,7 +94,6 @@ void RGWObjManifest::dump(Formatter *f) const
f->close_section();
f->dump_unsigned("obj_size", obj_size);
::encode_json("explicit_objs", explicit_objs, f);
::encode_json("head_obj", head_obj, f);
::encode_json("head_size", head_size, f);
::encode_json("max_head_size", max_head_size, f);
::encode_json("prefix", prefix, f);
@ -576,7 +575,7 @@ void rgw_bucket::dump(Formatter *f) const
encode_json("marker", marker, f);
encode_json("bucket_id", bucket_id, f);
encode_json("tenant", tenant, f);
encode_json("placement", placement, f);
encode_json("explicit_placement", explicit_placement, f);
}
void rgw_bucket::decode_json(JSONObj *obj) {
@ -584,12 +583,12 @@ void rgw_bucket::decode_json(JSONObj *obj) {
JSONDecoder::decode_json("marker", marker, obj);
JSONDecoder::decode_json("bucket_id", bucket_id, obj);
JSONDecoder::decode_json("tenant", tenant, obj);
JSONDecoder::decode_json("placement", placement, obj);
if (placement.data_pool.empty()) {
JSONDecoder::decode_json("explicit_placement", explicit_placement, obj);
if (explicit_placement.data_pool.empty()) {
/* decoding old format */
JSONDecoder::decode_json("pool", placement.data_pool, obj);
JSONDecoder::decode_json("data_extra_pool", placement.data_extra_pool, obj);
JSONDecoder::decode_json("index_pool", placement.index_pool, obj);
JSONDecoder::decode_json("pool", explicit_placement.data_pool, obj);
JSONDecoder::decode_json("data_extra_pool", explicit_placement.data_extra_pool, obj);
JSONDecoder::decode_json("index_pool", explicit_placement.index_pool, obj);
}
}

27
src/rgw/rgw_op.cc
View File

@ -2322,9 +2322,10 @@ void RGWCreateBucket::execute()
if (s->bucket_exists) {
string selected_placement_rule;
rgw_bucket bucket;
bucket.tenant = s->bucket_tenant;
bucket.name = s->bucket_name;
op_ret = store->select_bucket_placement(*(s->user), zonegroup_id,
placement_rule,
s->bucket_tenant, s->bucket_name,
bucket, &selected_placement_rule, nullptr);
if (selected_placement_rule != s->bucket_info.placement_rule) {
op_ret = -EEXIST;
@ -2517,7 +2518,7 @@ void RGWDeleteBucket::execute()
}
}
op_ret = rgw_bucket_sync_user_stats(store, s->user->user_id, s->bucket);
op_ret = rgw_bucket_sync_user_stats(store, s->user->user_id, s->bucket_info);
if ( op_ret < 0) {
ldout(s->cct, 1) << "WARNING: failed to sync user stats before bucket delete: op_ret= " << op_ret << dendl;
}
@ -2536,7 +2537,7 @@ void RGWDeleteBucket::execute()
}
}
op_ret = store->delete_bucket(s->bucket, ot);
op_ret = store->delete_bucket(s->bucket_info, ot);
if (op_ret == -ECANCELED) {
// lost a race, either with mdlog sync or another delete bucket operation.
@ -2617,11 +2618,6 @@ public:
int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand)
{
int r = prepare_init(store, NULL);
if (r < 0) {
return r;
}
string oid = obj_str;
upload_id = s->info.args.get("uploadId");
if (!oid_rand) {
@ -2659,15 +2655,20 @@ int RGWPutObjProcessor_Multipart::prepare(RGWRados *store, string *oid_rand)
manifest.set_multipart_part_rule(store->ctx()->_conf->rgw_obj_stripe_size, num);
r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, target_obj);
int r = manifest_gen.create_begin(store->ctx(), &manifest, bucket, target_obj);
if (r < 0) {
return r;
}
cur_obj = manifest_gen.get_cur_obj();
cur_obj = manifest_gen.get_cur_obj(store);
rgw_raw_obj_to_obj(bucket, cur_obj, &head_obj);
head_obj.index_hash_source = obj_str;
r = prepare_init(store, NULL);
if (r < 0) {
return r;
}
return 0;
}
@ -4920,7 +4921,7 @@ void RGWCompleteMultipart::execute()
op_ret = -ERR_INVALID_PART;
return;
} else {
manifest.append(obj_part.manifest);
manifest.append(obj_part.manifest, store);
}
if (obj_part.cs_info.compression_type != "none") {
@ -5079,7 +5080,7 @@ void RGWAbortMultipart::execute()
RGWObjManifest::obj_iterator oiter = obj_part.manifest.obj_begin();
if (oiter != obj_part.manifest.obj_end()) {
rgw_obj head;
rgw_raw_obj raw_head = oiter.get_location();
rgw_raw_obj raw_head = oiter.get_location().get_raw_obj(store);
rgw_raw_obj_to_obj(s->bucket, raw_head, &head);
rgw_obj_key key;
@ -5371,7 +5372,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path)
RGWObjVersionTracker ot;
ot.read_version = binfo.ep_objv;
ret = store->delete_bucket(binfo.bucket, ot);
ret = store->delete_bucket(binfo, ot);
if (0 == ret) {
ret = rgw_unlink_bucket(store, binfo.owner, binfo.bucket.tenant,
binfo.bucket.name, false);

2
src/rgw/rgw_orphan.cc
View File

@ -437,7 +437,7 @@ int RGWOrphanSearch::handle_stat_result(map<int, list<string> >& oids, RGWRados:
RGWObjManifest::obj_iterator miter;
for (miter = manifest.obj_begin(); miter != manifest.obj_end(); ++miter) {
const rgw_raw_obj& loc = miter.get_location();
const rgw_raw_obj& loc = miter.get_location().get_raw_obj(store);
string s = loc.oid;
obj_oids.insert(obj_fingerprint(s));
}

36
src/rgw/rgw_quota.cc
View File

@ -265,9 +265,19 @@ public:
int BucketAsyncRefreshHandler::init_fetch()
{
RGWBucketInfo bucket_info;
RGWObjectCtx obj_ctx(store);
int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
return r;
}
ldout(store->ctx(), 20) << "initiating async quota refresh for bucket=" << bucket << dendl;
int r = store->get_bucket_stats_async(bucket, RGW_NO_SHARD, this);
r = store->get_bucket_stats_async(bucket_info, RGW_NO_SHARD, this);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket.name << dendl;
@ -327,14 +337,22 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& user, rgw_buck
{
RGWBucketInfo bucket_info;
RGWObjectCtx obj_ctx(store);
int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
return r;
}
string bucket_ver;
string master_ver;
map<RGWObjCategory, RGWStorageStats> bucket_stats;
int r = store->get_bucket_stats(bucket, RGW_NO_SHARD, &bucket_ver,
r = store->get_bucket_stats(bucket_info, RGW_NO_SHARD, &bucket_ver,
&master_ver, bucket_stats, nullptr);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket="
ldout(store->ctx(), 0) << "could not get bucket stats for bucket="
<< bucket.name << dendl;
return r;
}
@ -576,7 +594,17 @@ int RGWUserStatsCache::fetch_stats_from_storage(const rgw_user& user, rgw_bucket
int RGWUserStatsCache::sync_bucket(const rgw_user& user, rgw_bucket& bucket)
{
int r = rgw_bucket_sync_user_stats(store, user, bucket);
RGWBucketInfo bucket_info;
RGWObjectCtx obj_ctx(store);
int r = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
if (r < 0) {
ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << " r=" << r << dendl;
return r;
}
r = rgw_bucket_sync_user_stats(store, user, bucket_info);
if (r < 0) {
ldout(store->ctx(), 0) << "ERROR: rgw_bucket_sync_user_stats() for user=" << user << ", bucket=" << bucket << " returned " << r << dendl;
return r;

458
src/rgw/rgw_rados.cc
View File

File diff suppressed because it is too large Load Diff

214
src/rgw/rgw_rados.h
View File

@ -36,6 +36,8 @@ class RGWMetaSyncProcessorThread;
class RGWDataSyncProcessorThread;
class RGWSyncLogTrimThread;
class RGWRESTConn;
struct RGWZoneGroup;
struct RGWZoneParams;
/* flags for put_obj_meta() */
#define PUT_OBJ_CREATE 0x01
@ -76,17 +78,6 @@ static inline void get_obj_bucket_and_oid_loc(const rgw_obj& obj, string& oid, s
int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset, RGWAccessControlPolicy *policy);
static inline void rgw_obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj)
{
get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc);
if (!obj.is_in_extra_data()) {
raw_obj->pool = obj.bucket.placement.data_pool;
} else {
raw_obj->pool = obj.bucket.placement.get_data_extra_pool();
}
}
static inline bool rgw_raw_obj_to_obj(const rgw_bucket& bucket, const rgw_raw_obj& raw_obj, rgw_obj *obj)
{
string name;
@ -108,6 +99,43 @@ static inline bool rgw_raw_obj_to_obj(const rgw_bucket& bucket, const rgw_raw_ob
return true;
}
bool rgw_obj_to_raw(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params,
const rgw_obj& obj, rgw_raw_obj *raw_obj);
class rgw_obj_select {
rgw_obj obj;
rgw_raw_obj raw_obj;
bool is_raw;
public:
rgw_obj_select() : is_raw(false) {}
rgw_obj_select(const rgw_obj& _obj) : obj(_obj), is_raw(false) {}
rgw_obj_select(const rgw_raw_obj& _raw_obj) : raw_obj(_raw_obj), is_raw(true) {}
rgw_obj_select(const rgw_obj_select& rhs) {
is_raw = rhs.is_raw;
if (is_raw) {
raw_obj = rhs.raw_obj;
} else {
obj = rhs.obj;
}
}
rgw_raw_obj get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const;
rgw_raw_obj get_raw_obj(RGWRados *store) const;
rgw_obj_select& operator=(const rgw_obj& rhs) {
obj = rhs;
is_raw = false;
return *this;
}
rgw_obj_select& operator=(const rgw_raw_obj& rhs) {
raw_obj = rhs;
is_raw = true;
return *this;
}
};
struct compression_block {
uint64_t old_ofs;
uint64_t new_ofs;
@ -338,6 +366,7 @@ struct RGWObjManifestRule {
};
WRITE_CLASS_ENCODER(RGWObjManifestRule)
class RGWObjManifest {
protected:
bool explicit_objs; /* old manifest? */
@ -346,7 +375,6 @@ protected:
uint64_t obj_size;
rgw_obj obj;
rgw_raw_obj head_obj; /* in-memory only, calculated from obj */
uint64_t head_size;
uint64_t max_head_size;
@ -357,8 +385,8 @@ protected:
string tail_instance; /* tail object's instance */
void convert_to_explicit();
int append_explicit(RGWObjManifest& m);
void convert_to_explicit(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params);
int append_explicit(RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params);
void append_rules(RGWObjManifest& m, map<uint64_t, RGWObjManifestRule>::iterator& iter, string *override_prefix);
void update_iterators() {
@ -377,7 +405,6 @@ public:
objs = rhs.objs;
obj_size = rhs.obj_size;
obj = rhs.obj;
head_obj = rhs.head_obj;
head_size = rhs.head_size;
max_head_size = rhs.max_head_size;
prefix = rhs.prefix;
@ -405,7 +432,7 @@ public:
objs.swap(_objs);
}
void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_raw_obj *location);
void get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location);
void set_trivial_rule(uint64_t tail_ofs, uint64_t stripe_max_size) {
RGWObjManifestRule rule(0, tail_ofs, 0, stripe_max_size);
@ -507,8 +534,6 @@ public:
tail_instance = obj.get_instance();
}
rgw_obj_to_raw(obj, &head_obj);
update_iterators();
DECODE_FINISH(bl);
}
@ -516,7 +541,8 @@ public:
void dump(Formatter *f) const;
static void generate_test_instances(list<RGWObjManifest*>& o);
int append(RGWObjManifest& m);
int append(RGWObjManifest& m, RGWZoneGroup& zonegroup, RGWZoneParams& zone_params);
int append(RGWObjManifest& m, RGWRados *store);
bool get_rule(uint64_t ofs, RGWObjManifestRule *rule);
@ -544,7 +570,6 @@ public:
void set_head(const rgw_obj& _o, uint64_t _s) {
obj = _o;
rgw_obj_to_raw(obj, &head_obj);
head_size = _s;
if (explicit_objs && head_size > 0) {
@ -557,10 +582,6 @@ public:
return obj;
}
const rgw_raw_obj& get_head() {
return head_obj;
}
void set_tail_bucket(const rgw_bucket& _b) {
tail_bucket = _b;
}
@ -622,7 +643,7 @@ public:
int cur_stripe;
string cur_override_prefix;
rgw_raw_obj location;
rgw_obj_select location;
map<uint64_t, RGWObjManifestRule>::iterator rule_iter;
map<uint64_t, RGWObjManifestRule>::iterator next_rule_iter;
@ -672,7 +693,7 @@ public:
bool operator!=(const obj_iterator& rhs) {
return (ofs != rhs.ofs);
}
const rgw_raw_obj& get_location() {
const rgw_obj_select& get_location() {
return location;
}
@ -731,7 +752,7 @@ public:
string oid_prefix;
rgw_raw_obj cur_obj;
rgw_obj_select cur_obj;
rgw_pool pool;
@ -744,7 +765,8 @@ public:
int create_next(uint64_t ofs);
const rgw_raw_obj& get_cur_obj() { return cur_obj; }
rgw_raw_obj get_cur_obj(RGWZoneGroup& zonegroup, RGWZoneParams& zone_params) { return cur_obj.get_raw_obj(zonegroup, zone_params); }
rgw_raw_obj get_cur_obj(RGWRados *store) { return cur_obj.get_raw_obj(store); }
/* total max size of current stripe (including head obj) */
uint64_t cur_stripe_max_size() {
@ -1032,8 +1054,6 @@ public:
};
WRITE_CLASS_ENCODER(RGWSystemMetaObj)
struct RGWZoneGroup;
struct RGWZonePlacementInfo {
string index_pool;
string data_pool;
@ -1196,6 +1216,69 @@ struct RGWZoneParams : RGWSystemMetaObj {
void dump(Formatter *f) const;
void decode_json(JSONObj *obj);
static void generate_test_instances(list<RGWZoneParams*>& o);
bool find_placement(const rgw_data_placement_target& placement, string *placement_id) {
for (const auto& pp : placement_pools) {
const RGWZonePlacementInfo& info = pp.second;
if (info.index_pool == placement.index_pool.to_str() &&
info.data_pool == placement.data_pool.to_str() &&
info.data_extra_pool == placement.data_extra_pool.to_str()) {
*placement_id = pp.first;
return true;
}
}
return false;
}
bool get_placement(const string& placement_id, RGWZonePlacementInfo *placement) const {
auto iter = placement_pools.find(placement_id);
if (iter == placement_pools.end()) {
return false;
}
*placement = iter->second;
return true;
}
bool get_index_pool(const rgw_obj& obj, rgw_pool *pool) {
if (!obj.bucket.explicit_placement.index_pool.empty()) {
*pool = obj.bucket.explicit_placement.index_pool;
return true;
}
if (obj.placement_id.empty()) {
return false;
}
auto iter = placement_pools.find(obj.placement_id);
if (iter == placement_pools.end()) {
return false;
}
pool->init(iter->second.index_pool);
return true;
}
bool get_data_pool(const rgw_obj& obj, rgw_pool *pool) const {
const rgw_data_placement_target& explicit_placement = obj.bucket.explicit_placement;
if (!explicit_placement.data_pool.empty()) {
if (!obj.in_extra_data) {
*pool = explicit_placement.data_pool;
} else {
*pool = explicit_placement.get_data_extra_pool();
}
return true;
}
if (obj.placement_id.empty()) {
return false;
}
auto iter = placement_pools.find(obj.placement_id);
if (iter == placement_pools.end()) {
return false;
}
if (!obj.in_extra_data) {
pool->init(iter->second.data_pool);
} else {
pool->init(iter->second.data_extra_pool);
}
return true;
}
};
WRITE_CLASS_ENCODER(RGWZoneParams)
@ -2063,24 +2146,24 @@ class RGWRados
int open_objexp_pool_ctx();
int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx);
int open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx& index_ctx);
int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, string& bucket_oid);
int open_bucket_index_base(rgw_bucket& bucket, librados::IoCtx& index_ctx,
int open_bucket_index_ctx(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx);
int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx, string& bucket_oid);
int open_bucket_index_base(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
string& bucket_oid_base);
int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
int open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
const string& obj_key, string *bucket_obj, int *shard_id);
int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
int open_bucket_index_shard(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
int shard_id, string *bucket_obj);
int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
map<int, string>& bucket_objs, int shard_id = -1, map<int, string> *bucket_instance_ids = NULL);
template<typename T>
int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
int open_bucket_index(const RGWBucketInfo& bucket_info, librados::IoCtx& index_ctx,
map<int, string>& oids, map<int, T>& bucket_objs,
int shard_id = -1, map<int, string> *bucket_instance_ids = NULL);
void build_bucket_index_marker(const string& shard_id_str, const string& shard_marker,
string *marker);
void get_bucket_instance_ids(RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result);
void get_bucket_instance_ids(const RGWBucketInfo& bucket_info, int shard_id, map<int, string> *result);
atomic64_t max_req_id;
Mutex lock;
@ -2334,16 +2417,14 @@ public:
}
int get_required_alignment(const rgw_pool& pool, uint64_t *alignment);
int get_required_alignment(const rgw_bucket& bucket, uint64_t *alignment);
int get_max_chunk_size(const rgw_pool& pool, uint64_t *max_chunk_size);
int get_max_chunk_size(const rgw_bucket& bucket, uint64_t *max_chunk_size);
int get_max_chunk_size(const rgw_obj& obj, uint64_t *max_chunk_size);
uint32_t get_max_bucket_shards() {
return MAX_BUCKET_INDEX_SHARDS_PRIME;
}
int get_raw_obj_ref(const rgw_raw_obj& obj, rgw_rados_ref *ref, rgw_pool *pool = NULL);
static void obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj);
int list_raw_objects(const rgw_pool& pool, const string& prefix_filter, int max,
RGWListRawObjsCtx& ctx, list<string>& oids,
@ -2411,18 +2492,18 @@ public:
* create a bucket with name bucket and the given list of attrs
* returns 0 on success, -ERR# otherwise.
*/
int init_bucket_index(rgw_bucket& bucket, int num_shards);
int init_bucket_index(RGWBucketInfo& bucket_info, int num_shards);
int select_bucket_placement(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
RGWZonePlacementInfo *rule_info);
int select_legacy_bucket_placement(const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
RGWZonePlacementInfo *rule_info);
rgw_bucket& bucket, string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
int select_legacy_bucket_placement(rgw_bucket& bucket, RGWZonePlacementInfo *rule_info);
int select_new_bucket_location(RGWUserInfo& user_info, const string& zonegroup_id, const string& rule,
const string& tenant_name, const string& bucket_name, rgw_bucket& bucket, string *pselected_rule_name,
RGWZonePlacementInfo *rule_info);
int set_bucket_location_by_rule(const string& location_rule, const string& tenant_name, const string& bucket_name, rgw_bucket& bucket,
RGWZonePlacementInfo *rule_info);
rgw_bucket& bucket, string *pselected_rule_name, RGWZonePlacementInfo *rule_info);
int select_bucket_location_by_rule(const string& location_rule, rgw_bucket& bucket, RGWZonePlacementInfo *rule_info);
void create_bucket_id(string *bucket_id);
bool get_obj_data_pool(const rgw_obj& obj, rgw_pool *pool);
bool obj_to_raw(const rgw_obj& obj, rgw_raw_obj *raw_obj);
int create_bucket(RGWUserInfo& owner, rgw_bucket& bucket,
const string& zonegroup_id,
const string& placement_rule,
@ -2972,14 +3053,14 @@ public:
* bucket: the name of the bucket to delete
* Returns 0 on success, -ERR# otherwise.
*/
int delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_tracker);
int delete_bucket(RGWBucketInfo& bucket_info, RGWObjVersionTracker& objv_tracker);
bool is_meta_master();
/**
* Check to see if the bucket metadata is synced
*/
bool is_syncing_bucket_meta(rgw_bucket& bucket);
bool is_syncing_bucket_meta(const rgw_bucket& bucket);
void wakeup_meta_sync_shards(set<int>& shard_ids);
void wakeup_data_sync_shards(const string& source_zone, map<int, set<string> >& shard_ids);
@ -3154,9 +3235,9 @@ public:
}
int decode_policy(bufferlist& bl, ACLOwner *owner);
int get_bucket_stats(rgw_bucket& bucket, int shard_id, string *bucket_ver, string *master_ver,
int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, string *bucket_ver, string *master_ver,
map<RGWObjCategory, RGWStorageStats>& stats, string *max_marker);
int get_bucket_stats_async(rgw_bucket& bucket, int shard_id, RGWGetBucketStats_CB *cb);
int get_bucket_stats_async(RGWBucketInfo& bucket_info, int shard_id, RGWGetBucketStats_CB *cb);
int get_user_stats(const rgw_user& user, RGWStorageStats& stats);
int get_user_stats_async(const rgw_user& user, RGWGetUserStats_CB *cb);
void get_bucket_instance_obj(const rgw_bucket& bucket, rgw_raw_obj& obj);
@ -3192,16 +3273,16 @@ public:
int cls_obj_complete_del(BucketShard& bs, string& tag, int64_t pool, uint64_t epoch, rgw_obj& obj,
ceph::real_time& removed_mtime, list<rgw_obj_key> *remove_objs, uint16_t bilog_flags);
int cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags);
int cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout);
int cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& start, const string& prefix,
int cls_obj_set_bucket_tag_timeout(RGWBucketInfo& bucket_info, uint64_t timeout);
int cls_bucket_list(RGWBucketInfo& bucket_info, int shard_id, rgw_obj_key& start, const string& prefix,
uint32_t num_entries, bool list_versions, map<string, RGWObjEnt>& m,
bool *is_truncated, rgw_obj_key *last_entry,
bool (*force_check_filter)(const string& name) = NULL);
int cls_bucket_head(rgw_bucket& bucket, int shard_id, map<string, struct rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL);
int cls_bucket_head_async(rgw_bucket& bucket, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio);
int list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max, std::list<rgw_bi_log_entry>& result, bool *truncated);
int trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, string& end_marker);
int get_bi_log_status(rgw_bucket& bucket, int shard_id, map<int, string>& max_marker);
int cls_bucket_head(const RGWBucketInfo& bucket_info, int shard_id, map<string, struct rgw_bucket_dir_header>& headers, map<int, string> *bucket_instance_ids = NULL);
int cls_bucket_head_async(const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio);
int list_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, uint32_t max, std::list<rgw_bi_log_entry>& result, bool *truncated);
int trim_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, string& end_marker);
int get_bi_log_status(RGWBucketInfo& bucket_info, int shard_id, map<int, string>& max_marker);
int bi_get_instance(rgw_obj& obj, rgw_bucket_dir_entry *dirent);
int bi_get(rgw_bucket& bucket, rgw_obj& obj, BIIndexType index_type, rgw_cls_bi_entry *entry);
@ -3280,11 +3361,11 @@ public:
int process_lc();
int list_lc_progress(const string& marker, uint32_t max_entries, map<string, int> *progress_map);
int bucket_check_index(rgw_bucket& bucket,
int bucket_check_index(RGWBucketInfo& bucket_info,
map<RGWObjCategory, RGWStorageStats> *existing_stats,
map<RGWObjCategory, RGWStorageStats> *calculated_stats);
int bucket_rebuild_index(rgw_bucket& bucket);
int remove_objs_from_index(rgw_bucket& bucket, list<rgw_obj_key>& oid_list);
int bucket_rebuild_index(RGWBucketInfo& bucket_info);
int remove_objs_from_index(RGWBucketInfo& bucket_info, list<rgw_obj_key>& oid_list);
int move_rados_obj(librados::IoCtx& src_ioctx,
const string& src_oid, const string& src_locator,
librados::IoCtx& dst_ioctx,
@ -3294,8 +3375,7 @@ public:
int cls_user_get_header(const string& user_id, cls_user_header *header);
int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);
int cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, rgw_bucket& bucket);
int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
int cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, const RGWBucketInfo& bucket_info);
int cls_user_list_buckets(rgw_raw_obj& obj,
const string& in_marker,
const string& end_marker,
@ -3368,7 +3448,7 @@ public:
librados::Rados* get_rados_handle();
int delete_raw_obj_aio(const rgw_raw_obj& obj, list<librados::AioCompletion *>& handles);
int delete_obj_aio(const rgw_obj& obj, rgw_bucket& bucket, RGWBucketInfo& info, RGWObjState *astate,
int delete_obj_aio(const rgw_obj& obj, RGWBucketInfo& info, RGWObjState *astate,
list<librados::AioCompletion *>& handles, bool keep_index_consistent);
private:
/**

6
src/rgw/rgw_rest_log.cc
View File

@ -414,7 +414,7 @@ void RGWOp_BILog_List::execute() {
send_response();
do {
list<rgw_bi_log_entry> entries;
int ret = store->list_bi_log_entries(bucket_info.bucket, shard_id,
int ret = store->list_bi_log_entries(bucket_info, shard_id,
marker, max_entries - count,
entries, &truncated);
if (ret < 0) {
@ -496,7 +496,7 @@ void RGWOp_BILog_Info::execute() {
}
}
map<RGWObjCategory, RGWStorageStats> stats;
int ret = store->get_bucket_stats(bucket_info.bucket, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
int ret = store->get_bucket_stats(bucket_info, shard_id, &bucket_ver, &master_ver, stats, &max_marker);
if (ret < 0 && ret != -ENOENT) {
http_ret = ret;
return;
@ -558,7 +558,7 @@ void RGWOp_BILog_Delete::execute() {
return;
}
}
http_ret = store->trim_bi_log_entries(bucket_info.bucket, shard_id, start_marker, end_marker);
http_ret = store->trim_bi_log_entries(bucket_info, shard_id, start_marker, end_marker);
if (http_ret < 0) {
dout(5) << "ERROR: trim_bi_log_entries() " << dendl;
}

10
src/rgw/rgw_user.cc
View File

@ -49,6 +49,7 @@ int rgw_user_sync_all_stats(RGWRados *store, const rgw_user& user_id)
bool is_truncated;
string marker;
int ret;
RGWObjectCtx obj_ctx(store);
do {
RGWUserBuckets user_buckets;
@ -65,7 +66,14 @@ int rgw_user_sync_all_stats(RGWRados *store, const rgw_user& user_id)
marker = i->first;
RGWBucketEnt& bucket_ent = i->second;
ret = rgw_bucket_sync_user_stats(store, user_id, bucket_ent.bucket);
RGWBucketInfo bucket_info;
ret = store->get_bucket_instance_info(obj_ctx, bucket_ent.bucket, bucket_info, NULL, NULL);
if (ret < 0) {
ldout(cct, 0) << "ERROR: could not read bucket info: bucket=" << bucket_ent.bucket << " ret=" << ret << dendl;
continue;
}
ret = rgw_bucket_sync_user_stats(store, user_id, bucket_info);
if (ret < 0) {
ldout(cct, 0) << "ERROR: could not sync bucket stats: ret=" << ret << dendl;
return ret;