diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 2f5711ecb32..b02668f3555 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -1427,7 +1427,7 @@ static int rgw_cls_gc_defer_entry(cls_method_context_t hctx, bufferlist *in, buf return gc_defer_entry(hctx, op.tag, op.expiration_secs); } -static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, +static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, bool expired_only, string& key_iter, uint32_t max_entries, bool *truncated, int (*cb)(cls_method_context_t, const string&, cls_rgw_gc_obj_info&, void *), void *param) @@ -1449,12 +1449,14 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, start_key = key_iter; } - utime_t now = ceph_clock_now(g_ceph_context); - string now_str; - get_time_key(now, &now_str); - prepend_index_prefix(now_str, GC_OBJ_TIME_INDEX, &end_key); + if (expired_only) { + utime_t now = ceph_clock_now(g_ceph_context); + string now_str; + get_time_key(now, &now_str); + prepend_index_prefix(now_str, GC_OBJ_TIME_INDEX, &end_key); - CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key.c_str()); + CLS_LOG(0, "gc_iterate_entries end_key=%s\n", end_key.c_str()); + } string filter; @@ -1475,7 +1477,7 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); - if (key.compare(end_key) >= 0) + if (!end_key.empty() && key.compare(end_key) >= 0) return 0; if (!key_in_index(key, GC_OBJ_TIME_INDEX)) @@ -1512,10 +1514,11 @@ static int gc_list_cb(cls_method_context_t hctx, const string& key, cls_rgw_gc_o } static int gc_list_entries(cls_method_context_t hctx, const string& marker, - uint32_t max, list& entries, bool *truncated) + uint32_t max, bool expired_only, + list& entries, bool *truncated) { string key_iter; - int ret = gc_iterate_entries(hctx, marker, + int ret = gc_iterate_entries(hctx, marker, expired_only, key_iter, max, truncated, gc_list_cb, &entries); return ret; @@ -1534,7 +1537,7 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist } cls_rgw_gc_list_ret op_ret; - int ret = gc_list_entries(hctx, op.marker, op.max, op_ret.entries, &op_ret.truncated); + int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, op_ret.entries, &op_ret.truncated); if (ret < 0) return ret; diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc index 2851f2bd702..a26d0afdf4c 100644 --- a/src/cls/rgw/cls_rgw_client.cc +++ b/src/cls/rgw/cls_rgw_client.cc @@ -324,13 +324,14 @@ void cls_rgw_gc_defer_entry(ObjectWriteOperation& op, uint32_t expiration_secs, op.exec("rgw", "gc_defer_entry", in); } -int cls_rgw_gc_list(IoCtx& io_ctx, string& oid, string& marker, uint32_t max, +int cls_rgw_gc_list(IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only, list& entries, bool *truncated) { bufferlist in, out; cls_rgw_gc_list_op call; call.marker = marker; call.max = max; + call.expired_only = expired_only; ::encode(call, in); int r = io_ctx.exec(oid, "rgw", "gc_list", in, out); if (r < 0) diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h index 39bb3c9fc4a..c6b5b757fa8 100644 --- a/src/cls/rgw/cls_rgw_client.h +++ b/src/cls/rgw/cls_rgw_client.h @@ -61,7 +61,7 @@ void cls_rgw_usage_log_add(librados::ObjectWriteOperation& op, rgw_usage_log_inf void cls_rgw_gc_set_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, cls_rgw_gc_obj_info& info); void cls_rgw_gc_defer_entry(librados::ObjectWriteOperation& op, uint32_t expiration_secs, const string& tag); -int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max, +int cls_rgw_gc_list(librados::IoCtx& io_ctx, string& oid, string& marker, uint32_t max, bool expired_only, list& entries, bool *truncated); void cls_rgw_gc_remove(librados::ObjectWriteOperation& op, const list& tags); diff --git a/src/cls/rgw/cls_rgw_ops.cc b/src/cls/rgw/cls_rgw_ops.cc index 2ffc53c92be..01c0666d9eb 100644 --- a/src/cls/rgw/cls_rgw_ops.cc +++ b/src/cls/rgw/cls_rgw_ops.cc @@ -49,6 +49,7 @@ void cls_rgw_gc_list_op::dump(Formatter *f) const { f->dump_string("marker", marker); f->dump_unsigned("max", max); + f->dump_bool("expired_only", expired_only); } void cls_rgw_gc_list_op::generate_test_instances(list& ls) diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h index 8f0ecfbdd26..e5db06010dc 100644 --- a/src/cls/rgw/cls_rgw_ops.h +++ b/src/cls/rgw/cls_rgw_ops.h @@ -345,20 +345,25 @@ WRITE_CLASS_ENCODER(cls_rgw_gc_defer_entry_op) struct cls_rgw_gc_list_op { string marker; uint32_t max; + bool expired_only; - cls_rgw_gc_list_op() : max(0) {} + cls_rgw_gc_list_op() : max(0), expired_only(true) {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(marker, bl); ::encode(max, bl); + ::encode(expired_only, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(marker, bl); ::decode(max, bl); + if (struct_v >= 2) { + ::decode(expired_only, bl); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 26da02e6779..ac192aaa6c7 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -87,7 +87,8 @@ void _usage() cerr << " usage trim trim usage (by user, date range)\n"; cerr << " temp remove remove temporary objects that were created up to\n"; cerr << " specified date (and optional time)\n"; - cerr << " gc list dump expired garbage collection objects\n"; + cerr << " gc list dump expired garbage collection objects (specify\n"; + cerr << " --include-all to list all entries, including unexpired)\n"; cerr << " gc process manually process garbage\n"; cerr << " metadata get get metadata info\n"; cerr << " metadata put put metadata info\n"; @@ -812,6 +813,7 @@ int main(int argc, char **argv) int64_t max_size = -1; bool have_max_objects = false; bool have_max_size = false; + int include_all = false; int sync_stats = false; @@ -952,6 +954,8 @@ int main(int argc, char **argv) // do nothing } else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) { // do nothing + } else if (ceph_argparse_binary_flag(args, i, &include_all, NULL, "--include-all", (char*)NULL)) { + // do nothing } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) { caps = val; } else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) { @@ -1891,7 +1895,7 @@ next: do { list result; - int ret = store->list_gc_objs(&index, marker, 1000, result, &truncated); + int ret = store->list_gc_objs(&index, marker, 1000, !include_all, result, &truncated); if (ret < 0) { cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl; return 1; diff --git a/src/rgw/rgw_gc.cc b/src/rgw/rgw_gc.cc index 1b2b8ca9afb..dab07f341ab 100644 --- a/src/rgw/rgw_gc.cc +++ b/src/rgw/rgw_gc.cc @@ -90,13 +90,13 @@ int RGWGC::remove(int index, const std::list& tags) return store->gc_operate(obj_names[index], &op); } -int RGWGC::list(int *index, string& marker, uint32_t max, std::list& result, bool *truncated) +int RGWGC::list(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated) { result.clear(); for (; *index < cct->_conf->rgw_gc_max_objs && result.size() < max; (*index)++, marker.clear()) { std::list entries; - int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), entries, truncated); + int ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[*index], marker, max - result.size(), expired_only, entries, truncated); if (ret == -ENOENT) continue; if (ret < 0) @@ -158,7 +158,7 @@ int RGWGC::process(int index, int max_secs) do { int max = 100; std::list entries; - ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, entries, &truncated); + ret = cls_rgw_gc_list(store->gc_pool_ctx, obj_names[index], marker, max, true, entries, &truncated); if (ret == -ENOENT) { ret = 0; goto done; diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h index b19afc55dd5..afaead49258 100644 --- a/src/rgw/rgw_gc.h +++ b/src/rgw/rgw_gc.h @@ -49,7 +49,7 @@ public: void initialize(CephContext *_cct, RGWRados *_store); void finalize(); - int list(int *index, string& marker, uint32_t max, std::list& result, bool *truncated); + int list(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated); void list_init(int *index) { *index = 0; } int process(int index, int process_max_secs); int process(); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index cce463b324b..f6c8a4fa8fe 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -5339,9 +5339,9 @@ int RGWRados::gc_operate(string& oid, librados::ObjectReadOperation *op, bufferl return gc_pool_ctx.operate(oid, op, pbl); } -int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, std::list& result, bool *truncated) +int RGWRados::list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated) { - return gc->list(index, marker, max, result, truncated); + return gc->list(index, marker, max, expired_only, result, truncated); } int RGWRados::process_gc() diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 64b9ef062ab..d3f9998e7cc 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1431,7 +1431,7 @@ public: int gc_aio_operate(string& oid, librados::ObjectWriteOperation *op); int gc_operate(string& oid, librados::ObjectReadOperation *op, bufferlist *pbl); - int list_gc_objs(int *index, string& marker, uint32_t max, std::list& result, bool *truncated); + int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated); int process_gc(); int defer_gc(void *ctx, rgw_obj& obj); diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index a7921e2b934..ca3e7471b10 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -48,7 +48,8 @@ usage trim trim usage (by user, date range) temp remove remove temporary objects that were created up to specified date (and optional time) - gc list dump expired garbage collection objects + gc list dump expired garbage collection objects (specify + --include-all to list all entries, including unexpired) gc process manually process garbage metadata get get metadata info metadata put put metadata info diff --git a/src/test/cls_rgw/test_cls_rgw.cc b/src/test/cls_rgw/test_cls_rgw.cc index 7a37deec105..44cb3030724 100644 --- a/src/test/cls_rgw/test_cls_rgw.cc +++ b/src/test/cls_rgw/test_cls_rgw.cc @@ -409,14 +409,14 @@ TEST(cls_rgw, gc_set) string marker; /* list chains, verify truncated */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, entries, &truncated)); + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 8, true, entries, &truncated)); ASSERT_EQ(8, (int)entries.size()); ASSERT_EQ(1, truncated); entries.clear(); /* list all chains, verify not truncated */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, entries, &truncated)); + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 10, true, entries, &truncated)); ASSERT_EQ(10, (int)entries.size()); ASSERT_EQ(0, truncated); @@ -483,7 +483,7 @@ TEST(cls_rgw, gc_defer) string marker; /* list chains, verify num entries as expected */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated)); ASSERT_EQ(1, (int)entries.size()); ASSERT_EQ(0, truncated); @@ -496,7 +496,7 @@ TEST(cls_rgw, gc_defer) entries.clear(); /* verify list doesn't show deferred entry (this may fail if cluster is thrashing) */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated)); ASSERT_EQ(0, (int)entries.size()); ASSERT_EQ(0, truncated); @@ -504,7 +504,7 @@ TEST(cls_rgw, gc_defer) sleep(5); /* verify list shows deferred entry */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated)); ASSERT_EQ(1, (int)entries.size()); ASSERT_EQ(0, truncated); @@ -519,7 +519,7 @@ TEST(cls_rgw, gc_defer) entries.clear(); /* verify entry was removed */ - ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, entries, &truncated)); + ASSERT_EQ(0, cls_rgw_gc_list(ioctx, oid, marker, 1, true, entries, &truncated)); ASSERT_EQ(0, (int)entries.size()); ASSERT_EQ(0, truncated);