RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-03-10 02:09:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

rgw-admin: command to list sync error logs

Browse Source 
$ radosgw-admin sync error list

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
This commit is contained in:
Yehuda Sadeh 2016-02-15 17:04:05 -08:00
parent 288501429e
commit 10cda14b51
5 changed files with 127 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
src/rgw/rgw_admin.cc
View File

@ -26,7 +26,7 @@
#include "rgw_rados.h"
#include "rgw_acl.h"
#include "rgw_acl_s3.h"
#include "rgw_log.h"
#include "rgw_formats.h"
#include "rgw_usage.h"
#include "rgw_replica_log.h"
@ -333,6 +333,7 @@ enum {
OPT_MDLOG_TRIM,
OPT_MDLOG_FETCH,
OPT_MDLOG_STATUS,
OPT_SYNC_ERROR_LIST,
OPT_BILOG_LIST,
OPT_BILOG_TRIM,
OPT_DATA_SYNC_STATUS,
@ -381,6 +382,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_
strcmp(cmd, "caps") == 0 ||
strcmp(cmd, "data") == 0 ||
strcmp(cmd, "datalog") == 0 ||
strcmp(cmd, "error") == 0 ||
strcmp(cmd, "gc") == 0 ||
strcmp(cmd, "key") == 0 ||
strcmp(cmd, "log") == 0 ||
@ -401,6 +403,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_
strcmp(cmd, "regionmap") == 0 ||
strcmp(cmd, "replicalog") == 0 ||
strcmp(cmd, "subuser") == 0 ||
strcmp(cmd, "sync") == 0 ||
strcmp(cmd, "temp") == 0 ||
strcmp(cmd, "usage") == 0 ||
strcmp(cmd, "user") == 0 ||
@ -664,6 +667,10 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_
return OPT_METADATA_SYNC_INIT;
if (strcmp(cmd, "run") == 0)
return OPT_METADATA_SYNC_RUN;
} else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "sync") == 0) &&
(strcmp(prev_cmd, "error") == 0)) {
if (strcmp(cmd, "list") == 0)
return OPT_SYNC_ERROR_LIST;
} else if (strcmp(prev_cmd, "mdlog") == 0) {
if (strcmp(cmd, "list") == 0)
return OPT_MDLOG_LIST;
@ -4731,41 +4738,78 @@ next:
}
}
if (opt_cmd == OPT_BILOG_LIST) {
if (bucket_name.empty()) {
cerr << "ERROR: bucket not specified" << std::endl;
return -EINVAL;
}
RGWBucketInfo bucket_info;
int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
if (ret < 0) {
cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
return -ret;
}
formatter->open_array_section("entries");
bool truncated;
int count = 0;
if (max_entries < 0)
if (opt_cmd == OPT_SYNC_ERROR_LIST) {
if (max_entries < 0) {
max_entries = 1000;
}
do {
list<rgw_bi_log_entry> entries;
ret = store->list_bi_log_entries(bucket, shard_id, marker, max_entries - count, entries, &truncated);
if (ret < 0) {
cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;
bool truncated;
utime_t start_time, end_time;
int ret = parse_date_str(start_date, start_time);
if (ret < 0)
return -ret;
ret = parse_date_str(end_date, end_time);
if (ret < 0)
return -ret;
if (shard_id < 0) {
shard_id = 0;
}
formatter->open_array_section("entries");
for (; shard_id < ERROR_LOGGER_SHARDS; ++shard_id) {
formatter->open_object_section("shard");
encode_json("shard_id", shard_id, formatter);
formatter->open_array_section("entries");
int count = 0;
string oid = RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX, shard_id);
do {
list<cls_log_entry> entries;
ret = store->time_log_list(oid, start_time, end_time, max_entries - count, entries,
marker, &marker, &truncated);
if (ret == -ENOENT) {
break;
}
if (ret < 0) {
cerr << "ERROR: store->time_log_list(): " << cpp_strerror(-ret) << std::endl;
return -ret;
}
count += entries.size();
for (auto& cls_entry : entries) {
rgw_sync_error_info log_entry;
auto iter = cls_entry.data.begin();
try {
::decode(log_entry, iter);
} catch (buffer::error& err) {
cerr << "ERROR: failed to decode log entry" << std::endl;
continue;
}
formatter->open_object_section("entry");
encode_json("id", cls_entry.id, formatter);
encode_json("section", cls_entry.section, formatter);
encode_json("name", cls_entry.name, formatter);
encode_json("timestamp", cls_entry.timestamp, formatter);
encode_json("info", log_entry, formatter);
formatter->close_section();
formatter->flush(cout);
}
} while (truncated && count < max_entries);
formatter->close_section();
formatter->close_section();
if (specified_shard_id) {
break;
}
count += entries.size();
for (list<rgw_bi_log_entry>::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
rgw_bi_log_entry& entry = *iter;
encode_json("entry", entry, formatter);
marker = entry.id;
}
formatter->flush(cout);
} while (truncated && count < max_entries);
}
formatter->close_section();
formatter->flush(cout);

3
src/rgw/rgw_data_sync.cc
View File

@ -684,7 +684,8 @@ public:
sync_status = retcode;
if (sync_status < 0) {
yield call(sync_env->error_logger->log_error_cr("data", bucket_name + ":" + bucket_instance, -sync_status, string("failed to sync bucket instance: ") + cpp_strerror(-sync_status)));
yield call(sync_env->error_logger->log_error_cr(sync_env->conn->get_remote_id(), "data", bucket_name + ":" + bucket_instance,
-sync_status, string("failed to sync bucket instance: ") + cpp_strerror(-sync_status)));
}
#warning what do do in case of error
if (!entry_marker.empty()) {

7
src/rgw/rgw_json_enc.cc
View File

@ -11,6 +11,7 @@
#include "rgw_keystone.h"
#include "rgw_basic_types.h"
#include "rgw_op.h"
#include "rgw_sync.h"
#include "common/ceph_json.h"
#include "common/Formatter.h"
@ -1189,3 +1190,9 @@ void rgw_meta_sync_status::dump(Formatter *f) const {
encode_json("info", sync_info, f);
encode_json("markers", sync_markers, f);
}
void rgw_sync_error_info::dump(Formatter *f) const {
encode_json("source_zone", source_zone, f);
encode_json("error_code", error_code, f);
encode_json("message", message, f);
}

40
src/rgw/rgw_sync.cc
View File

@ -25,42 +25,21 @@ static string mdlog_sync_status_oid = "mdlog.sync-status";
static string mdlog_sync_status_shard_prefix = "mdlog.sync-status.shard";
static string mdlog_sync_full_sync_index_prefix = "meta.full-sync.index";
struct rgw_sync_error_info {
uint32_t error_code;
string message;
rgw_sync_error_info() : error_code(0) {}
rgw_sync_error_info(uint32_t _error_code, const string& _message) : error_code(_error_code), message(_message) {}
void encode(bufferlist& bl) const {
ENCODE_START(1, 1, bl);
::encode(error_code, bl);
::encode(message, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
DECODE_START(1, bl);
::decode(error_code, bl);
::decode(message, bl);
DECODE_FINISH(bl);
}
};
WRITE_CLASS_ENCODER(rgw_sync_error_info)
RGWSyncErrorLogger::RGWSyncErrorLogger(RGWRados *_store, const string oid_prefix, int _num_shards) : store(_store), num_shards(_num_shards) {
char buf[oid_prefix.size() + 16];
for (int i = 0; i < num_shards; i++) {
snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), i);
oids.push_back(buf);
oids.push_back(get_shard_oid(oid_prefix, i));
}
}
string RGWSyncErrorLogger::get_shard_oid(const string& oid_prefix, int shard_id) {
char buf[oid_prefix.size() + 16];
snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), shard_id);
return string(buf);
}
RGWCoroutine *RGWSyncErrorLogger::log_error_cr(const string& section, const string& name, uint32_t error_code, const string& message) {
RGWCoroutine *RGWSyncErrorLogger::log_error_cr(const string& source_zone, const string& section, const string& name, uint32_t error_code, const string& message) {
cls_log_entry entry;
rgw_sync_error_info info(error_code, message);
rgw_sync_error_info info(source_zone, error_code, message);
bufferlist bl;
::encode(info, bl);
store->time_log_prepare_entry(entry, ceph_clock_now(store->ctx()), section, name, bl);
@ -1082,7 +1061,8 @@ int RGWMetaSyncSingleEntryCR::operate() {
if (sync_status < 0) {
ldout(sync_env->cct, 10) << *this << ": failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << dendl;
log_error() << "failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << std::endl;
yield call(sync_env->error_logger->log_error_cr(section, key, -sync_status, string("failed to read remote metadata entry: ") + cpp_strerror(-sync_status)));
yield call(sync_env->error_logger->log_error_cr(sync_env->conn->get_remote_id(), section, key, -sync_status,
string("failed to read remote metadata entry: ") + cpp_strerror(-sync_status)));
return set_cr_error(sync_status);
}

32
src/rgw/rgw_sync.h
View File

@ -33,9 +33,39 @@ class RGWSyncErrorLogger {
atomic_t counter;
public:
RGWSyncErrorLogger(RGWRados *_store, const string oid_prefix, int _num_shards);
RGWCoroutine *log_error_cr(const string& section, const string& name, uint32_t error_code, const string& message);
RGWCoroutine *log_error_cr(const string& source_zone, const string& section, const string& name, uint32_t error_code, const string& message);
static string get_shard_oid(const string& oid_prefix, int shard_id);
};
struct rgw_sync_error_info {
string source_zone;
uint32_t error_code;
string message;
rgw_sync_error_info() : error_code(0) {}
rgw_sync_error_info(const string& _source_zone, uint32_t _error_code, const string& _message) : source_zone(_source_zone), error_code(_error_code), message(_message) {}
void encode(bufferlist& bl) const {
ENCODE_START(1, 1, bl);
::encode(source_zone, bl);
::encode(error_code, bl);
::encode(message, bl);
ENCODE_FINISH(bl);
}
void decode(bufferlist::iterator& bl) {
DECODE_START(1, bl);
::decode(source_zone, bl);
::decode(error_code, bl);
::decode(message, bl);
DECODE_FINISH(bl);
}
void dump(Formatter *f) const;
};
WRITE_CLASS_ENCODER(rgw_sync_error_info)
#define DEFAULT_BACKOFF_MAX 30
class RGWSyncBackoff {