RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-24 11:37:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

rgw: log pool id, and store/retrieve pool id info

Browse Source
This commit is contained in:
Yehuda Sadeh 2011-06-13 15:27:38 -07:00
parent 34098a39e2
commit bc8e120939
14 changed files with 230 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/rgw/rgw_access.h
View File

@ -164,6 +164,8 @@ public:
virtual int set_attr(std::string& bucket, std::string& obj,
const char *name, bufferlist& bl) = 0;
virtual int get_bucket_id(std::string& bucket) { return -ENOTSUP; }
/**
* stat an object
*/

50
src/rgw/rgw_admin.cc
View File

@ -15,12 +15,17 @@ using namespace std;
#include "rgw_access.h"
#include "rgw_acl.h"
#include "rgw_log.h"
#include "rgw_formats.h"
#include "auth/Crypto.h"
#define SECRET_KEY_LEN 40
#define PUBLIC_ID_LEN 20
static RGWFormatter_Plain formatter_plain;
static RGWFormatter_XML formatter_xml;
static RGWFormatter_JSON formatter_json;
void usage()
{
cerr << "usage: radosgw_admin <cmd> [options...]" << std::endl;
@ -37,6 +42,7 @@ void usage()
cerr << " buckets list list buckets\n";
cerr << " bucket link link bucket to specified user\n";
cerr << " bucket unlink unlink bucket from specified user\n";
cerr << " pool info show pool information\n";
cerr << " policy read bucket/object policy\n";
cerr << " log show dump a log from specific object or (bucket + date)\n";
cerr << "options:\n";
@ -56,6 +62,9 @@ void usage()
cerr << " --bucket=<bucket>\n";
cerr << " --object=<object>\n";
cerr << " --date=<yyyy-mm-dd>\n";
cerr << " --pool-id=<pool-id>\n";
cerr << " --format=<format> specify output format for certain operations: xml,\n";
cerr << " json, plain\n";
generic_client_usage();
exit(1);
}
@ -75,6 +84,7 @@ enum {
OPT_BUCKET_LINK,
OPT_BUCKET_UNLINK,
OPT_POLICY,
OPT_POOL_INFO,
OPT_LOG_SHOW,
};
@ -143,6 +153,7 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
strcmp(cmd, "key") == 0 ||
strcmp(cmd, "buckets") == 0 ||
strcmp(cmd, "bucket") == 0 ||
strcmp(cmd, "pool") == 0 ||
strcmp(cmd, "log") == 0) {
*need_more = true;
return 0;
@ -186,6 +197,9 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
} else if (strcmp(prev_cmd, "log") == 0) {
if (strcmp(cmd, "show") == 0)
return OPT_LOG_SHOW;
} else if (strcmp(prev_cmd, "pool") == 0) {
if (strcmp(cmd, "info") == 0)
return OPT_POOL_INFO;
}
return -EINVAL;
@ -389,6 +403,9 @@ int main(int argc, char **argv)
char secret_key_buf[SECRET_KEY_LEN + 1];
char public_id_buf[PUBLIC_ID_LEN + 1];
bool user_modify_op;
int pool_id = 0;
const char *format = 0;
RGWFormatter *formatter = &formatter_xml;
FOR_EACH_ARG(args) {
if (CEPH_ARGPARSE_EQ("uid", 'i')) {
@ -422,6 +439,10 @@ int main(int argc, char **argv)
} else if (CEPH_ARGPARSE_EQ("access", '\0')) {
CEPH_ARGPARSE_SET_ARG_VAL(&access, OPT_STR);
perm_mask = str_to_perm(access);
} else if (CEPH_ARGPARSE_EQ("pool-id", '\0')) {
CEPH_ARGPARSE_SET_ARG_VAL(&pool_id, OPT_INT);
} else if (CEPH_ARGPARSE_EQ("format", '\0')) {
CEPH_ARGPARSE_SET_ARG_VAL(&format, OPT_STR);
} else {
if (!opt_cmd) {
opt_cmd = get_cmd(CEPH_ARGPARSE_VAL, prev_cmd, &need_more);
@ -443,6 +464,19 @@ int main(int argc, char **argv)
if (opt_cmd == OPT_NO_CMD)
usage();
if (format) {
if (strcmp(format, "xml") == 0)
formatter = &formatter_xml;
else if (strcmp(format, "json") == 0)
formatter = &formatter_json;
else if (strcmp(format, "plain") == 0)
formatter = &formatter_plain;
else {
cerr << "unrecognized format: " << format << std::endl;
usage();
}
}
if (subuser) {
char *suser = strdup(subuser);
char *p = strchr(suser, ':');
@ -825,5 +859,21 @@ int main(int argc, char **argv)
rgw_delete_user(info);
}
if (opt_cmd == OPT_POOL_INFO) {
RGWPoolInfo info;
int ret = rgw_retrieve_pool_info(pool_id, info);
if (ret < 0) {
cerr << "could not retrieve pool info for pool_id=" << pool_id << std::endl;
return ret;
}
formatter->init();
formatter->open_obj_section("Pool");
formatter->dump_value_int("ID", "%d", pool_id);
formatter->dump_value_str("Bucket", "%s", info.bucket.c_str());
formatter->dump_value_str("Owner", "%s", info.owner.c_str());
formatter->close_section("Pool");
formatter->flush(cout);
}
return 0;
}

25
src/rgw/rgw_common.cc
View File

@ -269,15 +269,30 @@ done_free:
free(p);
}
void RGWFormatter::flush()
void RGWFormatter::reset()
{
free(buf);
buf = NULL;
len = 0;
max_len = 0;
}
void RGWFormatter::flush(struct req_state *s)
{
if (!buf)
return;
RGW_LOG(0) << "flush(): buf='" << buf << "' strlen(buf)=" << strlen(buf) << dendl;
CGI_PutStr(s, buf, len - 1);
free(buf);
buf = NULL;
len = 0;
max_len = 0;
reset();
}
void RGWFormatter::flush(ostream& os)
{
if (!buf)
return;
RGW_LOG(0) << "flush(): buf='" << buf << "' strlen(buf)=" << strlen(buf) << dendl;
os << buf << std::endl;
reset();
}

30
src/rgw/rgw_common.h
View File

@ -275,11 +275,30 @@ struct RGWUserInfo
};
WRITE_CLASS_ENCODER(RGWUserInfo)
struct RGWPoolInfo
{
string bucket;
string owner;
void encode(bufferlist& bl) const {
__u32 ver = 1;
::encode(ver, bl);
::encode(bucket, bl);
::encode(owner, bl);
}
void decode(bufferlist::iterator& bl) {
__u32 ver;
::decode(ver, bl);
::decode(bucket, bl);
::decode(owner, bl);
}
};
WRITE_CLASS_ENCODER(RGWPoolInfo)
struct req_state;
class RGWFormatter {
protected:
struct req_state *s;
char *buf;
int len;
int max_len;
@ -288,8 +307,7 @@ protected:
public:
RGWFormatter() : buf(NULL), len(0), max_len(0) {}
virtual ~RGWFormatter() {}
void init(struct req_state *_s) {
s = _s;
void init() {
if (buf)
free(buf);
buf = NULL;
@ -297,8 +315,10 @@ public:
max_len = 0;
formatter_init();
}
void reset();
void write_data(const char *fmt, ...);
virtual void flush();
virtual void flush(struct req_state *s);
virtual void flush(ostream& os);
virtual int get_len() { return (len ? len - 1 : 0); } // don't include null termination in length
virtual void open_array_section(const char *name) = 0;
virtual void open_obj_section(const char *name) = 0;
@ -359,6 +379,8 @@ struct req_state {
utime_t time;
int pool_id;
req_state() : acl(NULL), os_auth_token(NULL), os_user(NULL), os_groups(NULL) {}
};

3
src/rgw/rgw_log.cc
View File

@ -57,6 +57,7 @@ int rgw_log_op(struct req_state *s)
entry.http_status = "200"; // default
entry.error_code = s->err.s3_code;
entry.pool_id = s->pool_id;
bufferlist bl;
::encode(entry, bl);
@ -68,7 +69,7 @@ int rgw_log_op(struct req_state *s)
localtime_r(&t, &bdt);
char buf[entry.bucket.size() + 16];
sprintf(buf, "%.4d-%.2d-%.2d-%s", (bdt.tm_year+1900), (bdt.tm_mon+1), bdt.tm_mday, entry.bucket.c_str());
sprintf(buf, "%.4d-%.2d-%.2d-%d-%s", (bdt.tm_year+1900), (bdt.tm_mon+1), bdt.tm_mday, s->pool_id, entry.bucket.c_str());
string oid = buf;
int ret = rgwstore->append_async(log_bucket, oid, bl.length(), bl);

9
src/rgw/rgw_log.h
View File

@ -4,7 +4,7 @@
#include "rgw_common.h"
#include "include/utime.h"
#define LOG_ENTRY_VER 2
#define LOG_ENTRY_VER 3
#define RGW_SHOULD_LOG_DEFAULT 1
@ -27,6 +27,7 @@ struct rgw_log_entry {
utime_t total_time;
string user_agent;
string referrer;
uint64_t pool_id;
void encode(bufferlist &bl) const {
uint8_t ver;
@ -48,6 +49,7 @@ struct rgw_log_entry {
::encode(user_agent, bl);
::encode(referrer, bl);
::encode(bytes_received, bl);
::encode(pool_id, bl);
}
void decode(bufferlist::iterator &p) {
uint8_t ver;
@ -71,6 +73,11 @@ struct rgw_log_entry {
::decode(bytes_received, p);
else
bytes_received = 0;
if (ver >= 3)
::decode(pool_id, p);
else
pool_id = -1;
}
};
WRITE_CLASS_ENCODER(rgw_log_entry)

12
src/rgw/rgw_op.cc
View File

@ -309,6 +309,7 @@ void RGWCreateBucket::execute()
bufferlist aclbl;
bool existed;
bool pol_ret;
int pool_id;
int r = get_policy_from_attr(&old_policy, rgw_root_bucket, s->bucket_str);
if (r >= 0) {
@ -343,6 +344,15 @@ void RGWCreateBucket::execute()
if (ret == -EEXIST)
ret = 0;
pool_id = rgwstore->get_bucket_id(s->bucket_str);
if (pool_id >= 0) {
s->pool_id = pool_id;
RGWPoolInfo info;
info.owner = s->user.user_id;
info.bucket = s->bucket_str;
rgw_store_pool_info(pool_id, info);
}
done:
send_response();
}
@ -833,6 +843,8 @@ int RGWHandler::do_read_permissions(bool only_bucket)
ret = -EACCES;
}
if (!s->bucket_str.empty())
s->pool_id = rgwstore->get_bucket_id(s->bucket_str);
return ret;
}

9
src/rgw/rgw_rados.cc
View File

@ -735,6 +735,15 @@ int RGWRados::obj_stat(std::string& bucket, std::string& obj, uint64_t *psize, t
return r;
}
int RGWRados::get_bucket_id(std::string& bucket)
{
librados::IoCtx io_ctx;
int r = open_bucket_ctx(bucket, io_ctx);
if (r < 0)
return r;
return io_ctx.get_id();
}
int RGWRados::tmap_set(std::string& bucket, std::string& obj, std::string& key, bufferlist& bl)
{
bufferlist cmdbl, emptybl;

2
src/rgw/rgw_rados.h
View File

@ -92,6 +92,8 @@ public:
virtual int obj_stat(std::string& bucket, std::string& obj, uint64_t *psize, time_t *pmtime);
virtual int get_bucket_id(std::string& bucket);
virtual bool supports_tmap() { return true; }
virtual int tmap_set(std::string& bucket, std::string& obj, std::string& key, bufferlist& bl);
virtual int tmap_create(std::string& bucket, std::string& obj, std::string& key, bufferlist& bl);

6
src/rgw/rgw_rest.cc
View File

@ -178,7 +178,7 @@ void end_header(struct req_state *s, const char *content_type)
dump_content_length(s, s->formatter->get_len());
}
CGI_PRINTF(s,"Content-type: %s\r\n\r\n", content_type);
s->formatter->flush();
s->formatter->flush(s);
s->header_ended = true;
}
@ -187,7 +187,7 @@ void abort_early(struct req_state *s, int err_no)
set_req_state_err(s, err_no);
dump_errno(s);
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
void dump_continue(struct req_state *s)
@ -437,7 +437,7 @@ void init_entities_from_header(struct req_state *s)
}
}
done:
s->formatter->init(s);
s->formatter->init();
}
static void line_unfold(const char *line, string& sdest)

16
src/rgw/rgw_rest_os.cc
View File

@ -46,7 +46,7 @@ void RGWListBuckets_REST_OS::send_response()
dump_content_length(s, s->formatter->get_len());
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
void RGWListBucket_REST_OS::send_response()
@ -105,7 +105,7 @@ void RGWListBucket_REST_OS::send_response()
s->formatter->close_section("container");
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
static void dump_container_metadata(struct req_state *s, RGWBucketEnt& bucket)
@ -128,7 +128,7 @@ void RGWStatBucket_REST_OS::send_response()
end_header(s);
dump_start(s);
s->formatter->flush();
s->formatter->flush(s);
}
void RGWCreateBucket_REST_OS::send_response()
@ -137,7 +137,7 @@ void RGWCreateBucket_REST_OS::send_response()
set_req_state_err(s, ret);
dump_errno(s);
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
void RGWDeleteBucket_REST_OS::send_response()
@ -149,7 +149,7 @@ void RGWDeleteBucket_REST_OS::send_response()
set_req_state_err(s, r);
dump_errno(s);
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
void RGWPutObj_REST_OS::send_response()
@ -160,7 +160,7 @@ void RGWPutObj_REST_OS::send_response()
set_req_state_err(s, ret);
dump_errno(s);
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
void RGWDeleteObj_REST_OS::send_response()
@ -172,7 +172,7 @@ void RGWDeleteObj_REST_OS::send_response()
set_req_state_err(s, r);
dump_errno(s);
end_header(s);
s->formatter->flush();
s->formatter->flush(s);
}
int RGWGetObj_REST_OS::send_response(void *handle)
@ -226,7 +226,7 @@ send_data:
if (get_data && !orig_ret) {
CGI_PutStr(s, data, len);
}
s->formatter->flush();
s->formatter->flush(s);
return 0;
}

6
src/rgw/rgw_rest_s3.cc
View File

@ -110,7 +110,7 @@ void RGWListBuckets_REST_S3::send_response()
list_all_buckets_end(s);
dump_content_length(s, s->formatter->get_len());
end_header(s, "application/xml");
s->formatter->flush();
s->formatter->flush(s);
}
void RGWListBucket_REST_S3::send_response()
@ -158,7 +158,7 @@ void RGWListBucket_REST_S3::send_response()
}
}
s->formatter->close_section("ListBucketResult");
s->formatter->flush();
s->formatter->flush(s);
}
void RGWCreateBucket_REST_S3::send_response()
@ -221,7 +221,7 @@ void RGWCopyObj_REST_S3::send_response()
}
}
s->formatter->close_section("CopyObjectResult");
s->formatter->flush();
s->formatter->flush(s);
}
}

107
src/rgw/rgw_user.cc
View File

@ -17,6 +17,8 @@ static string ui_email_bucket = USER_INFO_EMAIL_BUCKET_NAME;
static string ui_openstack_bucket = USER_INFO_OPENSTACK_BUCKET_NAME;
static string ui_uid_bucket = USER_INFO_UID_BUCKET_NAME;
static string pi_pool_bucket = POOL_INFO_BUCKET_NAME;
string rgw_root_bucket = RGW_ROOT_BUCKET;
#define READ_CHUNK_LEN (16 * 1024)
@ -46,6 +48,39 @@ static int put_obj(string& uid, string& bucket, string& oid, const char *data, s
return ret;
}
static int get_obj(string& bucket, string& key, bufferlist& bl)
{
int ret;
char *data = NULL;
struct rgw_err err;
RGWUID uid;
void *handle = NULL;
bufferlist::iterator iter;
int request_len = READ_CHUNK_LEN;
ret = rgwstore->prepare_get_obj(bucket, key, 0, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &handle, &err);
if (ret < 0)
return ret;
do {
ret = rgwstore->get_obj(&handle, bucket, key, &data, 0, request_len - 1);
if (ret < 0)
goto done;
if (ret < request_len)
break;
free(data);
request_len *= 2;
} while (true);
bl.append(data, ret);
free(data);
ret = 0;
done:
rgwstore->finish_get_obj(&handle);
return ret;
}
/**
* Save the given user information to storage.
* Returns: 0 on success, -ERR# on failure.
@ -117,40 +152,18 @@ int rgw_store_user_info(RGWUserInfo& info)
int rgw_get_user_info_from_index(string& key, string& bucket, RGWUserInfo& info)
{
bufferlist bl;
int ret;
char *data = NULL;
struct rgw_err err;
RGWUID uid;
void *handle = NULL;
bufferlist::iterator iter;
int request_len = READ_CHUNK_LEN;
ret = rgwstore->prepare_get_obj(bucket, key, 0, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, &handle, &err);
int ret = get_obj(bucket, key, bl);
if (ret < 0)
return ret;
do {
ret = rgwstore->get_obj(&handle, bucket, key, &data, 0, request_len - 1);
if (ret < 0)
goto done;
if (ret < request_len)
break;
free(data);
request_len *= 2;
} while (true);
bl.append(data, ret);
free(data);
iter = bl.begin();
bufferlist::iterator iter = bl.begin();
::decode(uid, iter);
if (!iter.end()) {
if (!iter.end())
info.decode(iter);
}
ret = 0;
done:
rgwstore->finish_get_obj(&handle);
return ret;
return 0;
}
/**
@ -429,3 +442,41 @@ int rgw_delete_user(RGWUserInfo& info) {
return 0;
}
int rgw_store_pool_info(int pool_id, RGWPoolInfo& pool_info)
{
bufferlist bl;
::encode(pool_info, bl);
string uid;
char buf[16];
snprintf(buf, sizeof(buf), "%d", pool_id);
string pool_id_str(buf);
int ret = put_obj(uid, pi_pool_bucket, pool_id_str, bl.c_str(), bl.length());
if (ret < 0) {
RGW_LOG(0) << "ERROR: could not write to pool=" << pi_pool_bucket << " obj=" << pool_id_str << " ret=" << ret << dendl;
}
return ret;
}
int rgw_retrieve_pool_info(int pool_id, RGWPoolInfo& pool_info)
{
bufferlist bl;
string uid;
char buf[16];
snprintf(buf, sizeof(buf), "%d", pool_id);
string pool_id_str(buf);
int ret = get_obj(pi_pool_bucket, pool_id_str, bl);
if (ret < 0) {
RGW_LOG(0) << "ERROR: could not read from pool=" << pi_pool_bucket << " obj=" << pool_id_str << " ret=" << ret << dendl;
return ret;
}
bufferlist::iterator iter = bl.begin();
::decode(pool_info, iter);
return 0;
}

6
src/rgw/rgw_user.h
View File

@ -14,6 +14,8 @@ using namespace std;
#define USER_INFO_UID_BUCKET_NAME ".users.uid"
#define RGW_USER_ANON_ID "anonymous"
#define POOL_INFO_BUCKET_NAME ".pool"
/**
* A string wrapper that includes encode/decode functions
* for easily accessing a UID in all forms
@ -142,4 +144,8 @@ extern int rgw_remove_uid_index(string& uid);
extern int rgw_remove_email_index(string& uid, string& email);
extern int rgw_remove_openstack_name_index(string& uid, string& openstack_name);
extern int rgw_store_pool_info(int pool_id, RGWPoolInfo& pool_info);
extern int rgw_retrieve_pool_info(int pool_id, RGWPoolInfo& pool_info);
#endif