RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-08 03:18:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'wip-rgw-sync-next' into next

Browse Source 
Reviewed-by: Yehuda Sadeh <yehuda@inktank.com>
This commit is contained in:
Josh Durgin 2013-10-24 11:38:13 -07:00
commit 9fa9ed122e
17 changed files with 132 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/cls/log/cls_log.cc
View File

@ -211,9 +211,8 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o
if (iter == keys.end())
done = true;
else
ret.marker = marker;
ret.marker = marker;
ret.truncated = !done;
::encode(ret, *out);

3
src/cls/log/cls_log_client.cc
View File

@ -107,7 +107,8 @@ public:
};
void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
string& in_marker, int max_entries, list<cls_log_entry>& entries,
const string& in_marker, int max_entries,
list<cls_log_entry>& entries,
string *out_marker, bool *truncated)
{
bufferlist inbl;

3
src/cls/log/cls_log_client.h
View File

@ -18,7 +18,8 @@ void cls_log_add(librados::ObjectWriteOperation& op, const utime_t& timestamp,
const string& section, const string& name, bufferlist& bl);
void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
string& in_marker, int max_entries, list<cls_log_entry>& entries,
const string& in_marker, int max_entries,
list<cls_log_entry>& entries,
string *out_marker, bool *truncated);
void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time,

23
src/rgw/rgw_admin.cc
View File

@ -468,27 +468,6 @@ ReplicaLogType get_replicalog_type(const string& name) {
return ReplicaLog_Invalid;
}
string escape_str(string& src, char c)
{
int pos = 0;
string dest;
do {
int new_pos = src.find(c, pos);
if (new_pos >= 0) {
dest += src.substr(pos, new_pos - pos);
dest += "\\";
dest += c;
} else {
dest += src.substr(pos);
return dest;
}
pos = new_pos + 1;
} while (pos < (int)src.size());
return dest;
}
static void show_user_info(RGWUserInfo& info, Formatter *formatter)
{
encode_json("user_info", info, formatter);
@ -1980,7 +1959,7 @@ next:
bool truncated;
do {
int ret = meta_log->list_entries(handle, 1000, entries, &truncated);
int ret = meta_log->list_entries(handle, 1000, entries, NULL, &truncated);
if (ret < 0) {
cerr << "ERROR: meta_log->list_entries(): " << cpp_strerror(-ret) << std::endl;
return -ret;

10
src/rgw/rgw_bucket.cc
View File

@ -1187,12 +1187,16 @@ int RGWDataChangesLog::add_entry(rgw_bucket& bucket) {
}
int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
list<rgw_data_change>& entries, string& marker, bool *truncated) {
list<rgw_data_change>& entries,
const string& marker,
string *out_marker,
bool *truncated) {
list<cls_log_entry> log_entries;
int ret = store->time_log_list(oids[shard], start_time, end_time,
max_entries, log_entries, marker, truncated);
max_entries, log_entries, marker,
out_marker, truncated);
if (ret < 0)
return ret;
@ -1220,7 +1224,7 @@ int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int
for (; marker.shard < num_shards && (int)entries.size() < max_entries;
marker.shard++, marker.marker.clear()) {
int ret = list_entries(marker.shard, start_time, end_time, max_entries - entries.size(), entries,
marker.marker, &truncated);
marker.marker, NULL, &truncated);
if (ret == -ENOENT) {
continue;
}

5
src/rgw/rgw_bucket.h
View File

@ -352,7 +352,10 @@ public:
int add_entry(rgw_bucket& bucket);
int renew_entries();
int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
list<rgw_data_change>& entries, string& marker, bool *truncated);
list<rgw_data_change>& entries,
const string& marker,
string *out_marker,
bool *truncated);
int trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
const string& start_marker, const string& end_marker);
int trim_entries(const utime_t& start_time, const utime_t& end_time,

52
src/rgw/rgw_common.cc
View File

@ -705,6 +705,58 @@ bool url_decode(string& src_str, string& dest_str)
return true;
}
static void escape_char(char c, string& dst)
{
char buf[16];
snprintf(buf, sizeof(buf), "%%%.2X", (unsigned int)c);
dst.append(buf);
}
static bool char_needs_url_encoding(char c)
{
if (c <= 0x20 || c >= 0x7f)
return true;
switch (c) {
case 0x22:
case 0x23:
case 0x25:
case 0x26:
case 0x2B:
case 0x2C:
case 0x2F:
case 0x3A:
case 0x3B:
case 0x3C:
case 0x3E:
case 0x3D:
case 0x3F:
case 0x40:
case 0x5B:
case 0x5D:
case 0x5C:
case 0x5E:
case 0x60:
case 0x7B:
case 0x7D:
return true;
}
return false;
}
void url_encode(const string& src, string& dst)
{
const char *p = src.c_str();
for (unsigned i = 0; i < src.size(); i++, p++) {
if (char_needs_url_encoding(*p)) {
escape_char(*p, dst);
continue;
}
dst.append(p, 1);
}
}
string rgw_trim_whitespace(const string& src)
{
if (src.empty()) {

1
src/rgw/rgw_common.h
View File

@ -1257,6 +1257,7 @@ extern bool verify_object_permission(struct req_state *s, int perm);
/** Convert an input URL into a sane object name
* by converting %-escaped strings into characters, etc*/
extern bool url_decode(string& src_str, string& dest_str);
extern void url_encode(const string& src, string& dst);
extern void calc_hmac_sha1(const char *key, int key_len,
const char *msg, int msg_len, char *dest);

10
src/rgw/rgw_metadata.cc
View File

@ -109,9 +109,10 @@ void RGWMetadataLog::complete_list_entries(void *handle) {
}
int RGWMetadataLog::list_entries(void *handle,
int max_entries,
list<cls_log_entry>& entries,
bool *truncated) {
int max_entries,
list<cls_log_entry>& entries,
string *last_marker,
bool *truncated) {
LogListCtx *ctx = static_cast<LogListCtx *>(handle);
if (!max_entries) {
@ -120,7 +121,8 @@ int RGWMetadataLog::list_entries(void *handle,
}
int ret = store->time_log_list(ctx->cur_oid, ctx->from_time, ctx->end_time,
max_entries, entries, ctx->marker, truncated);
max_entries, entries, ctx->marker,
last_marker, truncated);
if ((ret < 0) && (ret != -ENOENT))
return ret;

4
src/rgw/rgw_metadata.h
View File

@ -150,7 +150,9 @@ public:
void complete_list_entries(void *handle);
int list_entries(void *handle,
int max_entries,
list<cls_log_entry>& entries, bool *truncated);
list<cls_log_entry>& entries,
string *out_marker,
bool *truncated);
int trim(int shard_id, const utime_t& from_time, const utime_t& end_time, const string& start_marker, const string& end_marker);
int get_info(int shard_id, RGWMetadataLogInfo *info);

4
src/rgw/rgw_op.cc
View File

@ -269,7 +269,7 @@ static int read_policy(RGWRados *store, struct req_state *s,
string oid = object;
rgw_obj obj;
if (bucket_info.flags & BUCKET_SUSPENDED) {
if (!s->system_request && bucket_info.flags & BUCKET_SUSPENDED) {
ldout(s->cct, 0) << "NOTICE: bucket " << bucket_info.bucket.name << " is suspended" << dendl;
return -ERR_USER_SUSPENDED;
}
@ -292,7 +292,7 @@ static int read_policy(RGWRados *store, struct req_state *s,
if (ret < 0)
return ret;
string& owner = bucket_policy.get_owner().get_id();
if (owner.compare(s->user.user_id) != 0 &&
if (!s->system_request && owner.compare(s->user.user_id) != 0 &&
!bucket_policy.verify_permission(s->user.user_id, s->perm_mask, RGW_PERM_READ))
ret = -EACCES;
else

8
src/rgw/rgw_rados.cc
View File

@ -1586,7 +1586,10 @@ int RGWRados::time_log_add(const string& oid, list<cls_log_entry>& entries)
}
int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated)
int max_entries, list<cls_log_entry>& entries,
const string& marker,
string *out_marker,
bool *truncated)
{
librados::IoCtx io_ctx;
@ -1596,7 +1599,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end
return r;
librados::ObjectReadOperation op;
cls_log_list(op, start_time, end_time, marker, max_entries, entries, &marker, truncated);
cls_log_list(op, start_time, end_time, marker, max_entries, entries,
out_marker, truncated);
bufferlist obl;

3
src/rgw/rgw_rados.h
View File

@ -1382,7 +1382,8 @@ public:
int time_log_add(const string& oid, list<cls_log_entry>& entries);
int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl);
int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated);
int max_entries, list<cls_log_entry>& entries,
const string& marker, string *out_marker, bool *truncated);
int time_log_info(const string& oid, cls_log_header *header);
int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
const string& from_marker, const string& to_marker);

5
src/rgw/rgw_rest_client.cc
View File

@ -540,7 +540,10 @@ int RGWRESTStreamWriteRequest::complete(string& etag, time_t *mtime)
int RGWRESTStreamReadRequest::get_obj(RGWAccessKey& key, map<string, string>& extra_headers, rgw_obj& obj)
{
string resource = obj.bucket.name + "/" + obj.object;
string urlsafe_bucket, urlsafe_object;
url_encode(obj.bucket.name, urlsafe_bucket);
url_encode(obj.object, urlsafe_object);
string resource = urlsafe_bucket + "/" + urlsafe_object;
string new_url = url;
if (new_url[new_url.size() - 1] != '/')
new_url.append("/");

46
src/rgw/rgw_rest_log.cc
View File

@ -79,9 +79,9 @@ void RGWOp_MDLog_List::execute() {
meta_log->init_list_entries(shard_id, ut_st, ut_et, marker, &handle);
bool truncated;
do {
http_ret = meta_log->list_entries(handle, max_entries, entries, &truncated);
http_ret = meta_log->list_entries(handle, max_entries, entries,
&last_marker, &truncated);
if (http_ret < 0)
break;
@ -100,12 +100,18 @@ void RGWOp_MDLog_List::send_response() {
if (http_ret < 0)
return;
s->formatter->open_array_section("entries");
for (list<cls_log_entry>::iterator iter = entries.begin();
iter != entries.end(); ++iter) {
cls_log_entry& entry = *iter;
store->meta_mgr->dump_log_entry(entry, s->formatter);
flusher.flush();
s->formatter->open_object_section("log_entries");
s->formatter->dump_string("marker", last_marker);
s->formatter->dump_bool("truncated", truncated);
{
s->formatter->open_array_section("entries");
for (list<cls_log_entry>::iterator iter = entries.begin();
iter != entries.end(); ++iter) {
cls_log_entry& entry = *iter;
store->meta_mgr->dump_log_entry(entry, s->formatter);
flusher.flush();
}
s->formatter->close_section();
}
s->formatter->close_section();
flusher.flush();
@ -471,10 +477,12 @@ void RGWOp_DATALog_List::execute() {
}
}
bool truncated;
do {
// Note that last_marker is updated to be the marker of the last
// entry listed
http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et,
max_entries, entries, marker, &truncated);
max_entries, entries, marker,
&last_marker, &truncated);
if (http_ret < 0)
break;
@ -491,12 +499,18 @@ void RGWOp_DATALog_List::send_response() {
if (http_ret < 0)
return;
s->formatter->open_array_section("entries");
for (list<rgw_data_change>::iterator iter = entries.begin();
iter != entries.end(); ++iter) {
rgw_data_change& entry = *iter;
encode_json("entry", entry, s->formatter);
flusher.flush();
s->formatter->open_object_section("log_entries");
s->formatter->dump_string("marker", last_marker);
s->formatter->dump_bool("truncated", truncated);
{
s->formatter->open_array_section("entries");
for (list<rgw_data_change>::iterator iter = entries.begin();
iter != entries.end(); ++iter) {
rgw_data_change& entry = *iter;
encode_json("entry", entry, s->formatter);
flusher.flush();
}
s->formatter->close_section();
}
s->formatter->close_section();
flusher.flush();

8
src/rgw/rgw_rest_log.h
View File

@ -74,9 +74,11 @@ public:
class RGWOp_MDLog_List : public RGWRESTOp {
list<cls_log_entry> entries;
string last_marker;
bool truncated;
int http_ret;
public:
RGWOp_MDLog_List() : http_ret(0) {}
RGWOp_MDLog_List() : truncated(false), http_ret(0) {}
~RGWOp_MDLog_List() {}
int check_caps(RGWUserCaps& caps) {
@ -175,9 +177,11 @@ public:
class RGWOp_DATALog_List : public RGWRESTOp {
list<rgw_data_change> entries;
string last_marker;
bool truncated;
int http_ret;
public:
RGWOp_DATALog_List() : http_ret(0) {}
RGWOp_DATALog_List() : truncated(false), http_ret(0) {}
~RGWOp_DATALog_List() {}
int check_caps(RGWUserCaps& caps) {

60
src/rgw/rgw_rest_s3.cc
View File

@ -1085,59 +1085,6 @@ int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl)
return bl.length();
}
static void escape_char(char c, string& dst)
{
char buf[16];
snprintf(buf, sizeof(buf), "%%%.2X", (unsigned int)c);
dst.append(buf);
}
static bool char_needs_url_encoding(char c)
{
if (c < 0x20 || c >= 0x7f)
return true;
switch (c) {
case 0x20:
case 0x22:
case 0x23:
case 0x25:
case 0x26:
case 0x2B:
case 0x2C:
case 0x2F:
case 0x3A:
case 0x3B:
case 0x3C:
case 0x3E:
case 0x3D:
case 0x3F:
case 0x40:
case 0x5B:
case 0x5D:
case 0x5C:
case 0x5E:
case 0x60:
case 0x7B:
case 0x7D:
return true;
}
return false;
}
static void url_escape(const string& src, string& dst)
{
const char *p = src.c_str();
for (unsigned i = 0; i < src.size(); i++, p++) {
if (char_needs_url_encoding(*p)) {
escape_char(*p, dst);
continue;
}
dst.append(p, 1);
}
}
void RGWPostObj_ObjStore_S3::send_response()
{
if (ret == 0 && parts.count("success_action_redirect")) {
@ -1154,11 +1101,10 @@ void RGWPostObj_ObjStore_S3::send_response()
string etag_url;
url_escape(s->bucket_name_str, bucket);
url_escape(s->object_str, key);
url_escape(etag_str, etag_url);
url_encode(s->bucket_name_str, bucket);
url_encode(s->object_str, key);
url_encode(etag_str, etag_url);
redirect.append("?bucket=");
redirect.append(bucket);
redirect.append("&key=");