diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index a221d8f8d50..83d48fac188 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -26,7 +26,7 @@ #include "rgw_rados.h" #include "rgw_acl.h" #include "rgw_acl_s3.h" -#include "rgw_log.h" + #include "rgw_formats.h" #include "rgw_usage.h" #include "rgw_replica_log.h" @@ -333,6 +333,7 @@ enum { OPT_MDLOG_TRIM, OPT_MDLOG_FETCH, OPT_MDLOG_STATUS, + OPT_SYNC_ERROR_LIST, OPT_BILOG_LIST, OPT_BILOG_TRIM, OPT_DATA_SYNC_STATUS, @@ -381,6 +382,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_ strcmp(cmd, "caps") == 0 || strcmp(cmd, "data") == 0 || strcmp(cmd, "datalog") == 0 || + strcmp(cmd, "error") == 0 || strcmp(cmd, "gc") == 0 || strcmp(cmd, "key") == 0 || strcmp(cmd, "log") == 0 || @@ -401,6 +403,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_ strcmp(cmd, "regionmap") == 0 || strcmp(cmd, "replicalog") == 0 || strcmp(cmd, "subuser") == 0 || + strcmp(cmd, "sync") == 0 || strcmp(cmd, "temp") == 0 || strcmp(cmd, "usage") == 0 || strcmp(cmd, "user") == 0 || @@ -664,6 +667,10 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_ return OPT_METADATA_SYNC_INIT; if (strcmp(cmd, "run") == 0) return OPT_METADATA_SYNC_RUN; + } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "sync") == 0) && + (strcmp(prev_cmd, "error") == 0)) { + if (strcmp(cmd, "list") == 0) + return OPT_SYNC_ERROR_LIST; } else if (strcmp(prev_cmd, "mdlog") == 0) { if (strcmp(cmd, "list") == 0) return OPT_MDLOG_LIST; @@ -4731,41 +4738,78 @@ next: } } - if (opt_cmd == OPT_BILOG_LIST) { - if (bucket_name.empty()) { - cerr << "ERROR: bucket not specified" << std::endl; - return -EINVAL; - } - RGWBucketInfo bucket_info; - int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket); - if (ret < 0) { - cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; - return -ret; - } - formatter->open_array_section("entries"); - bool truncated; - int count = 0; - if (max_entries < 0) + if (opt_cmd == OPT_SYNC_ERROR_LIST) { + if (max_entries < 0) { max_entries = 1000; + } - do { - list entries; - ret = store->list_bi_log_entries(bucket, shard_id, marker, max_entries - count, entries, &truncated); - if (ret < 0) { - cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl; - return -ret; + bool truncated; + utime_t start_time, end_time; + + int ret = parse_date_str(start_date, start_time); + if (ret < 0) + return -ret; + + ret = parse_date_str(end_date, end_time); + if (ret < 0) + return -ret; + + if (shard_id < 0) { + shard_id = 0; + } + + formatter->open_array_section("entries"); + + for (; shard_id < ERROR_LOGGER_SHARDS; ++shard_id) { + formatter->open_object_section("shard"); + encode_json("shard_id", shard_id, formatter); + formatter->open_array_section("entries"); + + int count = 0; + string oid = RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX, shard_id); + + do { + list entries; + ret = store->time_log_list(oid, start_time, end_time, max_entries - count, entries, + marker, &marker, &truncated); + if (ret == -ENOENT) { + break; + } + if (ret < 0) { + cerr << "ERROR: store->time_log_list(): " << cpp_strerror(-ret) << std::endl; + return -ret; + } + + count += entries.size(); + + for (auto& cls_entry : entries) { + rgw_sync_error_info log_entry; + + auto iter = cls_entry.data.begin(); + try { + ::decode(log_entry, iter); + } catch (buffer::error& err) { + cerr << "ERROR: failed to decode log entry" << std::endl; + continue; + } + formatter->open_object_section("entry"); + encode_json("id", cls_entry.id, formatter); + encode_json("section", cls_entry.section, formatter); + encode_json("name", cls_entry.name, formatter); + encode_json("timestamp", cls_entry.timestamp, formatter); + encode_json("info", log_entry, formatter); + formatter->close_section(); + formatter->flush(cout); + } + } while (truncated && count < max_entries); + + formatter->close_section(); + formatter->close_section(); + + if (specified_shard_id) { + break; } - - count += entries.size(); - - for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) { - rgw_bi_log_entry& entry = *iter; - encode_json("entry", entry, formatter); - - marker = entry.id; - } - formatter->flush(cout); - } while (truncated && count < max_entries); + } formatter->close_section(); formatter->flush(cout); diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index c822d2d3995..9927670e39e 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -684,7 +684,8 @@ public: sync_status = retcode; if (sync_status < 0) { - yield call(sync_env->error_logger->log_error_cr("data", bucket_name + ":" + bucket_instance, -sync_status, string("failed to sync bucket instance: ") + cpp_strerror(-sync_status))); + yield call(sync_env->error_logger->log_error_cr(sync_env->conn->get_remote_id(), "data", bucket_name + ":" + bucket_instance, + -sync_status, string("failed to sync bucket instance: ") + cpp_strerror(-sync_status))); } #warning what do do in case of error if (!entry_marker.empty()) { diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 6bbee35ab1e..55a129bfd25 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -11,6 +11,7 @@ #include "rgw_keystone.h" #include "rgw_basic_types.h" #include "rgw_op.h" +#include "rgw_sync.h" #include "common/ceph_json.h" #include "common/Formatter.h" @@ -1189,3 +1190,9 @@ void rgw_meta_sync_status::dump(Formatter *f) const { encode_json("info", sync_info, f); encode_json("markers", sync_markers, f); } + +void rgw_sync_error_info::dump(Formatter *f) const { + encode_json("source_zone", source_zone, f); + encode_json("error_code", error_code, f); + encode_json("message", message, f); +} diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index 914e65b757d..2191ddbba2e 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -25,42 +25,21 @@ static string mdlog_sync_status_oid = "mdlog.sync-status"; static string mdlog_sync_status_shard_prefix = "mdlog.sync-status.shard"; static string mdlog_sync_full_sync_index_prefix = "meta.full-sync.index"; -struct rgw_sync_error_info { - uint32_t error_code; - string message; - - rgw_sync_error_info() : error_code(0) {} - rgw_sync_error_info(uint32_t _error_code, const string& _message) : error_code(_error_code), message(_message) {} - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - ::encode(error_code, bl); - ::encode(message, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); - ::decode(error_code, bl); - ::decode(message, bl); - DECODE_FINISH(bl); - } -}; -WRITE_CLASS_ENCODER(rgw_sync_error_info) - RGWSyncErrorLogger::RGWSyncErrorLogger(RGWRados *_store, const string oid_prefix, int _num_shards) : store(_store), num_shards(_num_shards) { - char buf[oid_prefix.size() + 16]; - for (int i = 0; i < num_shards; i++) { - snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), i); - oids.push_back(buf); + oids.push_back(get_shard_oid(oid_prefix, i)); } } +string RGWSyncErrorLogger::get_shard_oid(const string& oid_prefix, int shard_id) { + char buf[oid_prefix.size() + 16]; + snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), shard_id); + return string(buf); +} -RGWCoroutine *RGWSyncErrorLogger::log_error_cr(const string& section, const string& name, uint32_t error_code, const string& message) { +RGWCoroutine *RGWSyncErrorLogger::log_error_cr(const string& source_zone, const string& section, const string& name, uint32_t error_code, const string& message) { cls_log_entry entry; - rgw_sync_error_info info(error_code, message); + rgw_sync_error_info info(source_zone, error_code, message); bufferlist bl; ::encode(info, bl); store->time_log_prepare_entry(entry, ceph_clock_now(store->ctx()), section, name, bl); @@ -1082,7 +1061,8 @@ int RGWMetaSyncSingleEntryCR::operate() { if (sync_status < 0) { ldout(sync_env->cct, 10) << *this << ": failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << dendl; log_error() << "failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << std::endl; - yield call(sync_env->error_logger->log_error_cr(section, key, -sync_status, string("failed to read remote metadata entry: ") + cpp_strerror(-sync_status))); + yield call(sync_env->error_logger->log_error_cr(sync_env->conn->get_remote_id(), section, key, -sync_status, + string("failed to read remote metadata entry: ") + cpp_strerror(-sync_status))); return set_cr_error(sync_status); } diff --git a/src/rgw/rgw_sync.h b/src/rgw/rgw_sync.h index a9e6869d74d..e7d4b212df8 100644 --- a/src/rgw/rgw_sync.h +++ b/src/rgw/rgw_sync.h @@ -33,9 +33,39 @@ class RGWSyncErrorLogger { atomic_t counter; public: RGWSyncErrorLogger(RGWRados *_store, const string oid_prefix, int _num_shards); - RGWCoroutine *log_error_cr(const string& section, const string& name, uint32_t error_code, const string& message); + RGWCoroutine *log_error_cr(const string& source_zone, const string& section, const string& name, uint32_t error_code, const string& message); + + static string get_shard_oid(const string& oid_prefix, int shard_id); }; +struct rgw_sync_error_info { + string source_zone; + uint32_t error_code; + string message; + + rgw_sync_error_info() : error_code(0) {} + rgw_sync_error_info(const string& _source_zone, uint32_t _error_code, const string& _message) : source_zone(_source_zone), error_code(_error_code), message(_message) {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(source_zone, bl); + ::encode(error_code, bl); + ::encode(message, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(source_zone, bl); + ::decode(error_code, bl); + ::decode(message, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(rgw_sync_error_info) + #define DEFAULT_BACKOFF_MAX 30 class RGWSyncBackoff {