RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-03-30 07:19:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2120 from ceph/wip-8858

Browse Source 
Wip 8858

Reviewed-by: Josh Durgin <josh.durgin@inktank.com>
This commit is contained in:
Josh Durgin 2014-07-22 16:58:25 -07:00
commit 422218a3b3
9 changed files with 96 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/common/utf8.c
View File

@ -35,7 +35,7 @@ static int high_bits_set(int c)
* Assumes buf is of size MAX_UTF8_SZ
* Returns -1 on failure; number of bytes in the encoded value otherwise.
*/
static int encode_utf8(unsigned long u, unsigned char *buf)
int encode_utf8(unsigned long u, unsigned char *buf)
{
int i;
unsigned long max_val[MAX_UTF8_SZ] = {
@ -74,7 +74,7 @@ static int encode_utf8(unsigned long u, unsigned char *buf)
* Decode a UTF8 character from an array of bytes. Return character code.
* Upon error, return INVALID_UTF8_CHAR.
*/
static unsigned long decode_utf8(unsigned char *buf, int nbytes)
unsigned long decode_utf8(unsigned char *buf, int nbytes)
{
unsigned long code;
int i, j;

12
src/common/utf8.h
View File

@ -44,6 +44,18 @@ int check_for_control_characters(const char *buf, int len);
*/
int check_for_control_characters_cstr(const char *buf);
/* Encode a 31-bit UTF8 code point to 'buf'.
* Assumes buf is of size MAX_UTF8_SZ
* Returns -1 on failure; number of bytes in the encoded value otherwise.
*/
int encode_utf8(unsigned long u, unsigned char *buf);
/*
* Decode a UTF8 character from an array of bytes. Return character code.
* Upon error, return INVALID_UTF8_CHAR.
*/
unsigned long decode_utf8(unsigned char *buf, int nbytes);
#ifdef __cplusplus
}
#endif

2
src/rgw/rgw_admin.cc
View File

@ -1518,7 +1518,7 @@ int main(int argc, char **argv)
do {
list<rgw_bi_log_entry> entries;
ret = store->list_objects(bucket, max_entries - count, prefix, delim,
marker, result, common_prefixes, true,
marker, NULL, result, common_prefixes, true,
ns, false, &truncated, NULL);
if (ret < 0) {
cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl;

6
src/rgw/rgw_bucket.cc
View File

@ -372,7 +372,7 @@ int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& b
if (delete_children) {
int max = 1000;
ret = store->list_objects(bucket, max, prefix, delim, marker,
ret = store->list_objects(bucket, max, prefix, delim, marker, NULL,
objs, common_prefixes,
false, ns, true, NULL, NULL);
@ -388,7 +388,7 @@ int rgw_remove_bucket(RGWRados *store, const string& bucket_owner, rgw_bucket& b
}
objs.clear();
ret = store->list_objects(bucket, max, prefix, delim, marker, objs, common_prefixes,
ret = store->list_objects(bucket, max, prefix, delim, marker, NULL, objs, common_prefixes,
false, ns, true, NULL, NULL);
if (ret < 0)
return ret;
@ -641,7 +641,7 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state,
do {
vector<RGWObjEnt> result;
int r = store->list_objects(bucket, max, prefix, delim, marker,
int r = store->list_objects(bucket, max, prefix, delim, marker, NULL,
result, common_prefixes, false,
ns, true,
&is_truncated, NULL);

8
src/rgw/rgw_op.cc
View File

@ -706,7 +706,7 @@ static int iterate_user_manifest_parts(CephContext *cct, RGWRados *store, off_t
do {
#define MAX_LIST_OBJS 100
int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker,
int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker, NULL,
objs, common_prefixes,
true, no_ns, true, &is_truncated, NULL);
if (r < 0)
@ -1104,7 +1104,9 @@ void RGWListBucket::execute()
if (ret < 0)
return;
ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, objs, common_prefixes,
string *pnext_marker = (delimiter.empty() ? NULL : &next_marker);
ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, pnext_marker, objs, common_prefixes,
!!(s->prot_flags & RGW_REST_SWIFT), no_ns, true, &is_truncated, NULL);
}
@ -2993,7 +2995,7 @@ void RGWListBucketMultiparts::execute()
}
}
marker_meta = marker.get_meta();
ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, objs, common_prefixes,
ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, NULL, objs, common_prefixes,
!!(s->prot_flags & RGW_REST_SWIFT), mp_ns, true, &is_truncated, &mp_filter);
if (!objs.empty()) {
vector<RGWObjEnt>::iterator iter;

1
src/rgw/rgw_op.h
View File

@ -202,6 +202,7 @@ class RGWListBucket : public RGWOp {
protected:
string prefix;
string marker;
string next_marker;
string max_keys;
string delimiter;
int max;

72
src/rgw/rgw_rados.cc
View File

@ -6,6 +6,7 @@
#include <sys/types.h>
#include "common/ceph_json.h"
#include "common/utf8.h"
#include "common/errno.h"
#include "common/Formatter.h"
@ -2102,12 +2103,13 @@ int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset,
* here.
*/
int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string& delim,
string& marker, vector<RGWObjEnt>& result, map<string, bool>& common_prefixes,
string& marker, string *next_marker, vector<RGWObjEnt>& result,
map<string, bool>& common_prefixes,
bool get_content_type, string& ns, bool enforce_ns,
bool *is_truncated, RGWAccessListFilter *filter)
{
int count = 0;
bool truncated;
bool truncated = true;
if (bucket_is_system(bucket)) {
return -EINVAL;
@ -2123,9 +2125,39 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
prefix_obj.set_obj(prefix);
string cur_prefix = prefix_obj.object;
do {
string bigger_than_delim;
if (!delim.empty()) {
unsigned long val = decode_utf8((unsigned char *)delim.c_str(), delim.size());
char buf[delim.size() + 16];
int r = encode_utf8(val + 1, (unsigned char *)buf);
if (r < 0) {
ldout(cct,0) << "ERROR: encode_utf8() failed" << dendl;
return -EINVAL;
}
buf[r] = '\0';
bigger_than_delim = buf;
}
string skip_after_delim;
/* if marker points at a common prefix, fast forward it into its upperbound string */
if (!delim.empty()) {
int delim_pos = cur_marker.find(delim, prefix.size());
if (delim_pos >= 0) {
cur_marker = cur_marker.substr(0, delim_pos);
cur_marker.append(bigger_than_delim);
}
}
while (truncated && count <= max) {
if (skip_after_delim > cur_marker) {
cur_marker = skip_after_delim;
ldout(cct, 20) << "setting cur_marker=" << cur_marker << dendl;
}
std::map<string, RGWObjEnt> ent_map;
int r = cls_bucket_list(bucket, cur_marker, cur_prefix, max - count, ent_map,
int r = cls_bucket_list(bucket, cur_marker, cur_prefix, max + 1 - count, ent_map,
&truncated, &cur_marker);
if (r < 0)
return r;
@ -2148,6 +2180,10 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
continue;
}
if (next_marker && count < max) {
*next_marker = obj;
}
if (filter && !filter->filter(obj, key))
continue;
@ -2158,18 +2194,42 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string&
int delim_pos = obj.find(delim, prefix.size());
if (delim_pos >= 0) {
common_prefixes[obj.substr(0, delim_pos + 1)] = true;
string prefix_key = obj.substr(0, delim_pos + 1);
if (common_prefixes.find(prefix_key) == common_prefixes.end()) {
if (count >= max) {
truncated = true;
goto done;
}
if (next_marker) {
*next_marker = prefix_key;
}
common_prefixes[prefix_key] = true;
skip_after_delim = obj.substr(0, delim_pos);
skip_after_delim.append(bigger_than_delim);
ldout(cct, 20) << "skip_after_delim=" << skip_after_delim << dendl;
count++;
}
continue;
}
}
if (count >= max) {
truncated = true;
goto done;
}
RGWObjEnt ent = eiter->second;
ent.name = obj;
ent.ns = ns;
result.push_back(ent);
count++;
}
} while (truncated && count < max);
}
done:
if (is_truncated)

5
src/rgw/rgw_rados.h
View File

@ -1440,8 +1440,9 @@ public:
* here.
*/
virtual int list_objects(rgw_bucket& bucket, int max, std::string& prefix, std::string& delim,
std::string& marker, std::vector<RGWObjEnt>& result, map<string, bool>& common_prefixes,
bool get_content_type, string& ns, bool enforce_ns, bool *is_truncated, RGWAccessListFilter *filter);
std::string& marker, std::string *next_marker, std::vector<RGWObjEnt>& result,
map<string, bool>& common_prefixes, bool get_content_type, string& ns, bool enforce_ns,
bool *is_truncated, RGWAccessListFilter *filter);
virtual int create_pool(rgw_bucket& bucket);

5
src/rgw/rgw_rest_s3.cc
View File

@ -254,9 +254,10 @@ void RGWListBucket_ObjStore_S3::send_response()
s->formatter->open_object_section_in_ns("ListBucketResult",
"http://s3.amazonaws.com/doc/2006-03-01/");
s->formatter->dump_string("Name", s->bucket_name_str);
if (!prefix.empty())
s->formatter->dump_string("Prefix", prefix);
s->formatter->dump_string("Prefix", prefix);
s->formatter->dump_string("Marker", marker);
if (is_truncated && !next_marker.empty())
s->formatter->dump_string("NextMarker", next_marker);
s->formatter->dump_int("MaxKeys", max);
if (!delimiter.empty())
s->formatter->dump_string("Delimiter", delimiter);