Merge branch 'next'

This commit is contained in:
Sage Weil 2012-11-26 21:13:57 -08:00
commit a41dde3de5
19 changed files with 237 additions and 34 deletions

View File

@ -202,3 +202,25 @@ set automatically.
:Description: Whether rgw should use dns cname record of the request hostname field (if hostname is not equal to ``rgw dns name``)
:Type: Boolean
:Default: ``false``
``rgw enable ops log``
:Description: Whether rgw will log each successful operation
:Type: Boolean
:Default: ``true``
``rgw ops log rados``
:Description: Whether rgw operations logging should be written into the RADOS backend
:Type: Boolean
:Default: ``true``
``rgw ops log socket path``
:Description: The path of a unix domain socket to which operations logging data will be written.
:Type: String
:Default: N/A
``rgw ops log data backlog``
:Description: Total backlog data size for unix domain socket operations logging
:Type: Integer
:Default: ``5ul << 20``

View File

@ -224,7 +224,31 @@ For details on RADOS Gateway administration, see `radosgw-admin`_.
in quotes, or simply regenerating the key and ensuring that it
does not have an escape character.
Configuring the Operations Logging
==================================
By default, the RADOS Gateway will log every successful operation in the RADOS backend.
This means that every request, whether it is a read request or a write request will
generate a RADOS operation that writes data. This does not come without cost, and may
affect overall performance. Turning off logging completely can be done by adding the
following config option to ceph.conf::
rgw enable ops log = false
Another way to reduce the logging load is to send operations logging data to a unix domain
socket, instead of writing it to the RADOS backend::
rgw ops log rados = false
rgw enable ops log = true
rgw ops log socket path = <path to socket>
When specifying a unix domain socket, it is also possible to specify the maximum amount
of memory that will be used to keep the data backlog::
rgw ops log data backlog = <size in bytes>
Any backlogged data in excess to the specified size will be lost, so socket needs to be
constantly read.
Enabling Swift Access
=====================

View File

@ -1,12 +1,13 @@
#!/bin/sh
if [ ! -d ../.git ]; then
echo "not updating .git_version (no ../.git)"
dname=`dirname $0`
if [ ! -d $dname/../.git ]; then
echo "not updating .git_version (no $dname/../.git)"
exit 0
fi
dname=`dirname $0`
cur=`git rev-parse HEAD 2>/dev/null; git describe 2>/dev/null`
cur=`cd $dname && git rev-parse HEAD 2>/dev/null; git describe 2>/dev/null`
[ -e $1 ] && old=`cat $1`
if [ "$cur" != "$old" ]; then

View File

@ -5100,7 +5100,8 @@ int Client::getdir(const char *relpath, list<string>& contents)
/****** file i/o **********/
int Client::open(const char *relpath, int flags, mode_t mode)
int Client::open(const char *relpath, int flags, mode_t mode, int stripe_unit,
int stripe_count, int object_size, const char *data_pool)
{
ldout(cct, 3) << "open enter(" << relpath << ", " << flags << "," << mode << ") = " << dendl;
Mutex::Locker lock(client_lock);
@ -5124,7 +5125,8 @@ int Client::open(const char *relpath, int flags, mode_t mode)
r = path_walk(dirpath, &dir);
if (r < 0)
return r;
r = _create(dir, dname.c_str(), flags, mode, &in, &fh);
r = _create(dir, dname.c_str(), flags, mode, &in, &fh, stripe_unit,
stripe_count, object_size, data_pool);
created = true;
}
if (r < 0)
@ -5156,6 +5158,13 @@ int Client::open(const char *relpath, int flags, mode_t mode)
return r;
}
int Client::open(const char *relpath, int flags, mode_t mode)
{
/* Use default file striping parameters */
return open(relpath, flags, mode, file_stripe_unit, file_stripe_count,
object_size, NULL);
}
int Client::lookup_hash(inodeno_t ino, inodeno_t dirino, const char *name)
{
Mutex::Locker lock(client_lock);
@ -6598,7 +6607,8 @@ int Client::ll_mknod(vinodeno_t parent, const char *name, mode_t mode, dev_t rde
return r;
}
int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp, int uid, int gid)
int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp,
int stripe_unit, int stripe_count, int object_size, const char *data_pool, int uid, int gid)
{
ldout(cct, 3) << "_create(" << dir->ino << " " << name << ", 0" << oct << mode << dec << ")" << dendl;
@ -6622,8 +6632,8 @@ int Client::_create(Inode *dir, const char *name, int flags, mode_t mode, Inode
req->head.args.open.flags = flags | O_CREAT;
req->head.args.open.mode = mode;
req->head.args.open.stripe_unit = file_stripe_unit;
req->head.args.open.stripe_count = file_stripe_count;
req->head.args.open.stripe_unit = stripe_unit;
req->head.args.open.stripe_count = stripe_count;
req->head.args.open.object_size = object_size;
req->head.args.open.file_replication = file_replication;
req->dentry_drop = CEPH_CAP_FILE_SHARED;

View File

@ -529,7 +529,9 @@ private:
int _setxattr(Inode *in, const char *name, const void *value, size_t len, int flags, int uid=-1, int gid=-1);
int _removexattr(Inode *in, const char *nm, int uid=-1, int gid=-1);
int _open(Inode *in, int flags, mode_t mode, Fh **fhp, int uid=-1, int gid=-1);
int _create(Inode *in, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp, int uid=-1, int gid=-1);
int _create(Inode *in, const char *name, int flags, mode_t mode, Inode **inp, Fh **fhp,
int stripe_unit, int stripe_count, int object_size, const char *data_pool,
int uid=-1, int gid=-1);
loff_t _lseek(Fh *fh, loff_t offset, int whence);
int _read(Fh *fh, int64_t offset, uint64_t size, bufferlist *bl);
int _write(Fh *fh, int64_t offset, uint64_t size, const char *buf);
@ -620,6 +622,7 @@ public:
// file ops
int mknod(const char *path, mode_t mode, dev_t rdev=0);
int open(const char *path, int flags, mode_t mode=0);
int open(const char *path, int flags, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool);
int lookup_hash(inodeno_t ino, inodeno_t dirino, const char *name);
int lookup_ino(inodeno_t ino);
int close(int fd);

View File

@ -601,6 +601,25 @@ int ceph_mknod(struct ceph_mount_info *cmount, const char *path, mode_t mode, de
*/
int ceph_open(struct ceph_mount_info *cmount, const char *path, int flags, mode_t mode);
/**
* Create and/or open a file with a specific file layout.
*
* @param cmount the ceph mount handle to use for performing the open.
* @param path the path of the file to open. If the flags parameter includes O_CREAT,
* the file will first be created before opening.
* @param flags a set of option masks that control how the file is created/opened.
* @param mode the permissions to place on the file if the file does not exist and O_CREAT
* is specified in the flags.
* @param stripe_unit the stripe unit size (option, -1 for default)
* @param stripe_count the stripe count (optional, -1 for default)
* @param object_size the object size (optional, -1 for default)
* @param data_pool name of target data pool name (optional, NULL or empty string for default)
* @returns a non-negative file descriptor number on success or a negative error code on failure.
*/
int ceph_open_layout(struct ceph_mount_info *cmount, const char *path, int flags,
mode_t mode, int stripe_unit, int stripe_count, int object_size,
const char *data_pool);
/**
* Close the open file.
*

View File

@ -395,6 +395,27 @@ public class CephMount {
private static synchronized native int native_ceph_open(long mountp, String path, int flags, int mode);
/**
* Open a file with a specific file layout.
*
* @param path Path of file to open or create.
* @param flags Open flags.
* @param mode Permission mode.
* @param stripe_unit File layout stripe unit size.
* @param stripe_count File layout stripe count.
* @param object_size Size of each object.
* @param data_pool The target data pool.
* @return File descriptor.
*/
public int open(String path, int flags, int mode, int stripe_unit, int stripe_count,
int object_size, String data_pool) throws FileNotFoundException {
return native_ceph_open_layout(instance_ptr, path, flags, mode, stripe_unit,
stripe_count, object_size, data_pool);
}
private static synchronized native int native_ceph_open_layout(long mountp, String path,
int flags, int mode, int stripe_unit, int stripe_count, int object_size, String data_pool);
/**
* Close an open file.
*

View File

@ -1411,6 +1411,58 @@ JNIEXPORT jint JNICALL Java_com_ceph_fs_CephMount_native_1ceph_1open
return ret;
}
/*
* Class: com_ceph_fs_CephMount
* Method: native_ceph_open_layout
* Signature: (JLjava/lang/String;IIIIILjava/lang/String;)I
*/
JNIEXPORT jint JNICALL Java_com_ceph_fs_CephMount_native_1ceph_1open_1layout
(JNIEnv *env, jclass clz, jlong j_mntp, jstring j_path, jint j_flags, jint j_mode,
jint stripe_unit, jint stripe_count, jint object_size, jstring j_data_pool)
{
struct ceph_mount_info *cmount = get_ceph_mount(j_mntp);
CephContext *cct = ceph_get_mount_context(cmount);
const char *c_path, *c_data_pool = NULL;
int ret, flags = fixup_open_flags(j_flags);
CHECK_ARG_NULL(j_path, "@path is null", -1);
CHECK_MOUNTED(cmount, -1);
c_path = env->GetStringUTFChars(j_path, NULL);
if (!c_path) {
cephThrowInternal(env, "Failed to pin memory");
return -1;
}
if (j_data_pool) {
c_data_pool = env->GetStringUTFChars(j_data_pool, NULL);
if (!c_data_pool) {
env->ReleaseStringUTFChars(j_path, c_path);
cephThrowInternal(env, "Failed to pin memory");
return -1;
}
}
ldout(cct, 10) << "jni: open_layout: path " << c_path << " flags " << flags
<< " mode " << (int)j_mode << " stripe_unit " << stripe_unit
<< " stripe_count " << stripe_count << " object_size " << object_size
<< " data_pool " << (c_data_pool ? c_data_pool : "<NULL>") << dendl;
ret = ceph_open_layout(cmount, c_path, flags, (int)j_mode,
(int)stripe_unit, (int)stripe_count, (int)object_size, c_data_pool);
ldout(cct, 10) << "jni: open_layout: exit ret " << ret << dendl;
env->ReleaseStringUTFChars(j_path, c_path);
if (j_data_pool)
env->ReleaseStringUTFChars(j_data_pool, c_data_pool);
if (ret < 0)
handle_error(env, ret);
return ret;
}
/*
* Class: com_ceph_fs_CephMount
* Method: native_ceph_close

View File

@ -623,6 +623,15 @@ public class CephMountTest {
mount.unlink(path);
}
@Test
public void test_open_layout() throws Exception {
String path = makePath();
int fd = mount.open(path, CephMount.O_WRONLY|CephMount.O_CREAT, 0,
(1<<20), 1, (1<<20), null);
mount.close(fd);
mount.unlink(path);
}
/*
* open/close
*/

View File

@ -94,6 +94,11 @@ public class CephUnmountedTest {
mount.open("/a/path", 0, 0);
}
@Test(expected=CephNotMountedException.class)
public void test_open_layout() throws Exception {
mount.open("/a/path", 0, 0, 0, 0, 0, null);
}
@Test(expected=CephNotMountedException.class)
public void test_close() throws Exception {
mount.close(0);

View File

@ -610,6 +610,15 @@ extern "C" int ceph_open(struct ceph_mount_info *cmount, const char *path,
return cmount->get_client()->open(path, flags, mode);
}
extern "C" int ceph_open_layout(struct ceph_mount_info *cmount, const char *path, int flags,
mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool)
{
if (!cmount->is_mounted())
return -ENOTCONN;
return cmount->get_client()->open(path, flags, mode, stripe_unit,
stripe_count, object_size, data_pool);
}
extern "C" int ceph_close(struct ceph_mount_info *cmount, int fd)
{
if (!cmount->is_mounted())

View File

@ -4,11 +4,11 @@
compress
sharedscripts
postrotate
if [ -x `which invoke-rc.d` ]; then
if which invoke-rc.d && [ -x `which invoke-rc.d` ]; then
invoke-rc.d ceph reload >/dev/null
elif [ -x `which service` ]; then
elif which service && [ -x `which service` ]; then
service ceph reload >/dev/null
elif [ -x `which initctl` ]; then
elif which initctl && [ -x `which initctl` ]; then
# upstart reload isn't very helpful here:
# https://bugs.launchpad.net/upstart/+bug/1012938
for type in mon osd mds; do

View File

@ -734,7 +734,9 @@ void ObjectCacher::bh_write(BufferHead *bh)
{
assert(lock.is_locked());
ldout(cct, 7) << "bh_write " << *bh << dendl;
bh->ob->get();
// finishers
C_WriteCommit *oncommit = new C_WriteCommit(this, bh->ob->oloc.pool,
bh->ob->get_soid(), bh->start(), bh->length());
@ -842,6 +844,8 @@ void ObjectCacher::bh_write_commit(int64_t poolid, sobject_t oid, loff_t start,
// is the entire object set now clean and fully committed?
ObjectSet *oset = ob->oset;
ob->put();
if (flush_set_callback &&
was_dirty_or_tx > 0 &&
oset->dirty_or_tx == 0) { // nothing dirty/tx

View File

@ -182,8 +182,6 @@ class ObjectCacher {
int dirty_or_tx;
map< tid_t, list<Context*> > waitfor_commit;
list<Context*> waitfor_rd;
list<Context*> waitfor_wr;
public:
Object(const Object& other);
@ -215,11 +213,9 @@ class ObjectCacher {
void set_object_locator(object_locator_t& l) { oloc = l; }
bool can_close() {
if (data.empty() &&
waitfor_commit.empty() &&
waitfor_rd.empty() && waitfor_wr.empty() &&
dirty_or_tx == 0) {
assert(lru_is_expireable());
if (lru_is_expireable()) {
assert(data.empty());
assert(waitfor_commit.empty());
return true;
}
return false;

View File

@ -582,15 +582,6 @@ int RGWPutACLs_ObjStore::get_params()
return ret;
}
int RGWInitMultipart_ObjStore::get_params()
{
if (!s->args.exists("uploads")) {
ret = -ENOTSUP;
}
return ret;
}
static int read_all_chunked_input(req_state *s, char **pdata, int *plen)
{
#define READ_CHUNK 4096

View File

@ -152,8 +152,6 @@ class RGWInitMultipart_ObjStore : public RGWInitMultipart {
public:
RGWInitMultipart_ObjStore() {}
~RGWInitMultipart_ObjStore() {}
virtual int get_params();
};
class RGWCompleteMultipart_ObjStore : public RGWCompleteMultipart {

View File

@ -1237,7 +1237,7 @@ int RGWInitMultipart_ObjStore_S3::get_params()
policy = s3policy;
return RGWInitMultipart_ObjStore::get_params();
return 0;
}
void RGWInitMultipart_ObjStore_S3::send_response()
@ -1578,8 +1578,11 @@ RGWOp *RGWHandler_ObjStore_Obj_S3::op_post()
{
if (s->args.exists("uploadId"))
return new RGWCompleteMultipart_ObjStore_S3;
else
if (s->args.exists("uploads"))
return new RGWInitMultipart_ObjStore_S3;
return NULL;
}
int RGWHandler_ObjStore_S3::init_from_header(struct req_state *s, int default_formatter, bool configurable_format)

View File

@ -2,6 +2,8 @@
#include "rgw_usage.h"
#include "rgw_rest_usage.h"
#include "include/str_list.h"
#define dout_subsys ceph_subsys_rgw
class RGWOp_Usage_Get : public RGWRESTOp {
@ -31,6 +33,18 @@ void RGWOp_Usage_Get::execute() {
RESTArgs::get_bool(s, "show-entries", true, &show_entries);
RESTArgs::get_bool(s, "show-summary", true, &show_summary);
string cat_str;
RESTArgs::get_string(s, "categories", cat_str, &cat_str);
if (!cat_str.empty()) {
list<string> cat_list;
list<string>::iterator iter;
get_str_list(cat_str, cat_list);
for (iter = cat_list.begin(); iter != cat_list.end(); ++iter) {
categories[*iter] = true;
}
}
http_ret = RGWUsage::show(store, uid, start, end, show_entries, show_summary, &categories, flusher);
}

View File

@ -149,6 +149,28 @@ TEST(LibCephFS, Mount) {
ceph_shutdown(cmount);
}
TEST(LibCephFS, Open_layout) {
struct ceph_mount_info *cmount;
ASSERT_EQ(ceph_create(&cmount, NULL), 0);
ASSERT_EQ(ceph_conf_read_file(cmount, NULL), 0);
ASSERT_EQ(ceph_mount(cmount, NULL), 0);
/* valid layout */
char test_layout_file[256];
sprintf(test_layout_file, "test_layout_%d_b", getpid());
int fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 7, (1<<20), NULL);
ASSERT_GT(fd, 0);
ceph_close(cmount, fd);
/* invalid layout */
sprintf(test_layout_file, "test_layout_%d_c", getpid());
fd = ceph_open_layout(cmount, test_layout_file, O_CREAT, 0666, (1<<20), 1, 19, NULL);
ASSERT_EQ(fd, -EINVAL);
ceph_close(cmount, fd);
ceph_shutdown(cmount);
}
TEST(LibCephFS, Dir_ls) {
pid_t mypid = getpid();