RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2024-12-18 09:25:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #21755 from linuxbox2/wip-unordered-gc-delay

Browse Source 
rgw: use partial-order bucket listing in RGWLC, add configurable processing delay

Reviewed-by: J. Eric Ivancich <ivancich@redhat.com>
Reviewed-by: Daniel Gryniewicz <dang@redhat.com>
This commit is contained in:
Casey Bodley 2018-05-22 11:20:31 -04:00 committed by GitHub
commit 51b1542083
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/common/options.cc
View File

@ -5049,6 +5049,10 @@ std::vector<Option> get_rgw_options() {
.set_default(60)
.set_description(""),
Option("rgw_lc_thread_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
.set_default(0)
.set_description("Delay after processing of bucket listing chunks (i.e., per 1000 entries) in milliseconds"),
Option("rgw_lc_max_objs", Option::TYPE_INT, Option::LEVEL_ADVANCED)
.set_default(32)
.set_description("Number of lifecycle data shards")

16
src/rgw/rgw_lc.cc
View File

@ -293,7 +293,12 @@ int RGWLC::handle_multipart_expiration(RGWRados::Bucket *target, const map<strin
int ret;
RGWBucketInfo& bucket_info = target->get_bucket_info();
RGWRados::Bucket::List list_op(target);
auto delay_ms = cct->_conf->get_val<int64_t>("rgw_lc_thread_delay");
list_op.params.list_versions = false;
/* lifecycle processing does not depend on total order, so can
* take advantage of unorderd listing optimizations--such as
* operating on one shard at a time */
list_op.params.allow_unordered = true;
list_op.params.ns = RGW_OBJ_NS_MULTIPART;
list_op.params.filter = &mp_filter;
for (auto prefix_iter = prefix_map.begin(); prefix_iter != prefix_map.end(); ++prefix_iter) {
@ -328,7 +333,8 @@ int RGWLC::handle_multipart_expiration(RGWRados::Bucket *target, const map<strin
if (going_down())
return 0;
}
}
} /* for objs */
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
} while(is_truncated);
}
return 0;
@ -353,6 +359,7 @@ int RGWLC::bucket_lc_process(string& shard_id)
vector<rgw_bucket_dir_entry> objs;
RGWObjectCtx obj_ctx(store);
vector<std::string> result;
auto delay_ms = cct->_conf->get_val<int64_t>("rgw_lc_thread_delay");
boost::split(result, shard_id, boost::is_any_of(":"));
string bucket_tenant = result[0];
string bucket_name = result[1];
@ -396,7 +403,11 @@ int RGWLC::bucket_lc_process(string& shard_id)
ceph_clock_now() < ceph::real_clock::to_time_t(*prefix_iter->second.expiration_date)) {
continue;
}
/* lifecycle processing does not depend on total order, so can
* take advantage of unorderd listing optimizations--such as
* operating on one shard at a time */
list_op.params.prefix = prefix_iter->first;
list_op.params.allow_unordered = true;
do {
objs.clear();
list_op.params.marker = list_op.get_next_marker();
@ -470,7 +481,8 @@ int RGWLC::bucket_lc_process(string& shard_id)
if (going_down())
return 0;
}
}
} /* for objs */
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
} while (is_truncated);
}
} else {