diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index f6cc4b97bc3..b6fdd1ccca6 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -907,6 +907,123 @@ int check_min_obj_stripe_size(RGWRados *store, RGWBucketInfo& bucket_info, rgw_o return 0; } + +int check_obj_locator_underscore(RGWBucketInfo& bucket_info, rgw_obj& obj, rgw_obj_key& key, bool fix, bool remove_bad, Formatter *f) { + f->open_object_section("object"); + f->open_object_section("key"); + f->dump_string("name", key.name); + f->dump_string("instance", key.instance); + f->close_section(); + + string oid; + string locator; + + get_obj_bucket_and_oid_loc(obj, obj.bucket, oid, locator); + + f->dump_string("oid", oid); + f->dump_string("locator", locator); + + + RGWObjectCtx obj_ctx(store); + + RGWRados::Object op_target(store, bucket_info, obj_ctx, obj); + RGWRados::Object::Read read_op(&op_target); + + int ret = read_op.prepare(NULL, NULL); + bool needs_fixing = (ret == -ENOENT); + + f->dump_bool("needs_fixing", needs_fixing); + + string status = (needs_fixing ? "needs_fixing" : "ok"); + + if ((needs_fixing || remove_bad) && fix) { + ret = store->fix_head_obj_locator(obj.bucket, needs_fixing, remove_bad, key); + if (ret < 0) { + cerr << "ERROR: fix_head_object_locator() returned ret=" << ret << std::endl; + goto done; + } + status = "fixed"; + } + +done: + f->dump_string("status", status); + + f->close_section(); + + return 0; +} + +int do_check_object_locator(const string& bucket_name, bool fix, bool remove_bad, Formatter *f) +{ + if (remove_bad && !fix) { + cerr << "ERROR: can't have remove_bad specified without fix" << std::endl; + return -EINVAL; + } + + RGWBucketInfo bucket_info; + rgw_bucket bucket; + string bucket_id; + + f->open_object_section("bucket"); + f->dump_string("bucket", bucket_name); + int ret = init_bucket(bucket_name, bucket_id, bucket_info, bucket); + if (ret < 0) { + cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; + return ret; + } + bool truncated; + int count = 0; + + int max_entries = 1000; + + string prefix; + string delim; + vector result; + map common_prefixes; + string ns; + + RGWRados::Bucket target(store, bucket); + RGWRados::Bucket::List list_op(&target); + + string marker; + + list_op.params.prefix = prefix; + list_op.params.delim = delim; + list_op.params.marker = rgw_obj_key(marker); + list_op.params.ns = ns; + list_op.params.enforce_ns = false; + list_op.params.list_versions = true; + + f->open_array_section("check_objects"); + do { + ret = list_op.list_objects(max_entries - count, &result, &common_prefixes, &truncated); + if (ret < 0) { + cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl; + return -ret; + } + + count += result.size(); + + list objs; + for (vector::iterator iter = result.begin(); iter != result.end(); ++iter) { + rgw_obj_key key = iter->key; + rgw_obj obj(bucket, key); + + if (key.name[0] == '_') { + ret = check_obj_locator_underscore(bucket_info, obj, key, fix, remove_bad, f); + /* ignore return code, move to the next one */ + } + } + f->flush(cout); + } while (truncated && count < max_entries); + f->close_section(); + f->close_section(); + + f->flush(cout); + + return 0; +} + int main(int argc, char **argv) { vector args; @@ -944,6 +1061,8 @@ int main(int argc, char **argv) int yes_i_really_mean_it = false; int delete_child_objects = false; int fix = false; + int remove_bad = false; + int check_head_obj_locator = false; int max_buckets = -1; map categories; string caps; @@ -1128,6 +1247,10 @@ int main(int argc, char **argv) // do nothing } else if (ceph_argparse_binary_flag(args, i, &fix, NULL, "--fix", (char*)NULL)) { // do nothing + } else if (ceph_argparse_binary_flag(args, i, &remove_bad, NULL, "--remove-bad", (char*)NULL)) { + // do nothing + } else if (ceph_argparse_binary_flag(args, i, &check_head_obj_locator, NULL, "--check-head-obj-locator", (char*)NULL)) { + // do nothing } else if (ceph_argparse_binary_flag(args, i, &check_objects, NULL, "--check-objects", (char*)NULL)) { // do nothing } else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) { @@ -2348,7 +2471,15 @@ next: } if (opt_cmd == OPT_BUCKET_CHECK) { - RGWBucketAdminOp::check_index(store, bucket_op, f); + if (check_head_obj_locator) { + if (bucket_name.empty()) { + cerr << "ERROR: need to specify bucket name" << std::endl; + return EINVAL; + } + do_check_object_locator(bucket_name, fix, remove_bad, formatter); + } else { + RGWBucketAdminOp::check_index(store, bucket_op, f); + } } if (opt_cmd == OPT_BUCKET_RM) { diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index a1ae4d975f2..1baf1786252 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1228,6 +1228,15 @@ public: void reset_loc() { loc.clear(); + /* + * For backward compatibility. Older versions used to have object locator on all objects, + * however, the orig_obj was the effective object locator. This had the same effect as not + * having object locator at all for most objects but the ones that started with underscore as + * these were escaped. + */ + if (orig_obj[0] == '_') { + loc = orig_obj; + } } bool have_null_instance() { @@ -1265,6 +1274,7 @@ public: object.append("_"); object.append(o); } + reset_loc(); } /* diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 30af667193b..8ef9c7e7908 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2967,6 +2967,89 @@ int RGWRados::get_obj_ref(const rgw_obj& obj, rgw_rados_ref *ref, rgw_bucket *bu return 0; } +/* + * fixes an issue where head objects were supposed to have a locator created, but ended + * up without one + */ +int RGWRados::fix_head_obj_locator(rgw_bucket& bucket, bool copy_obj, bool remove_bad, rgw_obj_key& key) +{ + string oid; + string locator; + + rgw_obj obj(bucket, key); + + get_obj_bucket_and_oid_loc(obj, bucket, oid, locator); + + if (locator.empty()) { + ldout(cct, 20) << "object does not have a locator, nothing to fix" << dendl; + return 0; + } + + librados::IoCtx ioctx; + + int ret = get_obj_ioctx(obj, &ioctx); + if (ret < 0) { + cerr << "ERROR: get_obj_ioctx() returned ret=" << ret << std::endl; + return ret; + } + ioctx.locator_set_key(string()); /* override locator for this object, use empty locator */ + + uint64_t size; + time_t mtime; + bufferlist data; + + map attrs; + librados::ObjectReadOperation op; + op.getxattrs(&attrs, NULL); + op.stat(&size, &mtime, NULL); +#define HEAD_SIZE 512 * 1024 + op.read(0, HEAD_SIZE, &data, NULL); + + ret = ioctx.operate(oid, &op, NULL); + if (ret < 0) { + lderr(cct) << "ERROR: ioctx.operate(oid=" << oid << ") returned ret=" << ret << dendl; + return ret; + } + + if (size > HEAD_SIZE) { + lderr(cct) << "ERROR: returned object size (" << size << ") > HEAD_SIZE (" << HEAD_SIZE << ")" << dendl; + return -EIO; + } + + if (size != data.length()) { + lderr(cct) << "ERROR: returned object size (" << size << ") != data.length() (" << data.length() << ")" << dendl; + return -EIO; + } + + if (copy_obj) { + librados::ObjectWriteOperation wop; + + wop.mtime(&mtime); + + map::iterator iter; + for (iter = attrs.begin(); iter != attrs.end(); ++iter) { + wop.setxattr(iter->first.c_str(), iter->second); + } + + wop.write(0, data); + + ioctx.locator_set_key(locator); + ioctx.operate(oid, &wop); + } + + if (remove_bad) { + ioctx.locator_set_key(string()); + + ret = ioctx.remove(oid); + if (ret < 0) { + lderr(cct) << "ERROR: failed to remove original bad object" << dendl; + return ret; + } + } + + return 0; +} + int RGWRados::BucketShard::init(rgw_bucket& _bucket, rgw_obj& obj) { bucket = _bucket; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 30ca28c6f5b..01821a36538 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2032,6 +2032,7 @@ public: map *calculated_stats); int bucket_rebuild_index(rgw_bucket& bucket); int remove_objs_from_index(rgw_bucket& bucket, list& oid_list); + int fix_head_obj_locator(rgw_bucket& bucket, bool copy_obj, bool remove_bad, rgw_obj_key& key); int cls_user_get_header(const string& user_id, cls_user_header *header); int cls_user_get_header_async(const string& user_id, RGWGetUserHeader_CB *ctx);