mars/mars_device_aio.c

600 lines
15 KiB
C
Raw Normal View History

2010-11-26 13:45:10 +00:00
// (c) 2010 Thomas Schoebel-Theuer / 1&1 Internet AG
//#define BRICK_DEBUGGING
//#define MARS_DEBUGGING
2011-02-23 20:48:06 +00:00
//#define IO_DEBUGGING
2010-11-26 13:45:10 +00:00
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/list.h>
#include <linux/types.h>
#include <linux/blkdev.h>
#include <linux/kthread.h>
#include <linux/spinlock.h>
#include <linux/wait.h>
#include <linux/mmu_context.h>
#include <linux/file.h>
#include "mars.h"
#define MARS_MAX_AIO 1024
#define MARS_MAX_AIO_READ 32
///////////////////////// own type definitions ////////////////////////
#include "mars_device_aio.h"
////////////////// own brick / input / output operations //////////////////
2010-12-15 12:13:18 +00:00
static int device_aio_ref_get(struct device_aio_output *output, struct mref_object *mref)
2010-11-26 13:45:10 +00:00
{
_CHECK_ATOMIC(&mref->ref_count, !=, 0);
2010-12-10 17:40:20 +00:00
/* Buffered IO is implemented, but should not be used
* except for testing.
* Always precede this with a buf brick -- otherwise you
* can get bad performance!
2010-11-26 13:45:10 +00:00
*/
2010-12-10 17:40:20 +00:00
if (!mref->ref_data) {
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *mref_a = device_aio_mref_get_aspect(output, mref);
2010-12-10 17:40:20 +00:00
if (!mref_a)
return -EILSEQ;
mref->ref_data = kmalloc(mref->ref_len, GFP_MARS);
if (!mref->ref_data)
return -ENOMEM;
mref->ref_flags = 0;
mref_a->do_dealloc = true;
2011-02-23 20:48:06 +00:00
#if 0 // litter flags for testing
2010-12-10 17:40:20 +00:00
if (mref->ref_rw) {
static int random = 0;
if (!(random++ % 2))
2010-12-15 12:13:18 +00:00
mref->ref_flags |= MREF_UPTODATE;
2010-12-10 17:40:20 +00:00
}
#endif
}
2010-11-26 13:45:10 +00:00
atomic_inc(&mref->ref_count);
return 0;
}
2010-12-15 12:13:18 +00:00
static void device_aio_ref_put(struct device_aio_output *output, struct mref_object *mref)
2010-11-26 13:45:10 +00:00
{
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *mref_a;
2010-11-26 13:45:10 +00:00
CHECK_ATOMIC(&mref->ref_count, 1);
if (!atomic_dec_and_test(&mref->ref_count))
return;
2010-12-15 12:13:18 +00:00
mref_a = device_aio_mref_get_aspect(output, mref);
2010-12-10 17:40:20 +00:00
if (mref_a && mref_a->do_dealloc) {
kfree(mref->ref_data);
}
2010-12-15 12:13:18 +00:00
device_aio_free_mref(mref);
2010-11-26 13:45:10 +00:00
}
2010-12-15 12:13:18 +00:00
static void device_aio_ref_io(struct device_aio_output *output, struct mref_object *mref)
2010-11-26 13:45:10 +00:00
{
struct aio_threadinfo *tinfo = &output->tinfo[0];
struct generic_callback *cb = mref->ref_cb;
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *mref_a;
2010-11-26 13:45:10 +00:00
unsigned long flags;
atomic_inc(&mref->ref_count);
if (unlikely(!output->filp)) {
cb->cb_error = -EINVAL;
goto done;
}
2011-02-23 20:48:06 +00:00
MARS_IO("AIO rw=%d pos=%lld len=%d data=%p\n", mref->ref_rw, mref->ref_pos, mref->ref_len, mref->ref_data);
2010-11-26 13:45:10 +00:00
2010-12-15 12:13:18 +00:00
mref_a = device_aio_mref_get_aspect(output, mref);
2010-11-26 13:45:10 +00:00
traced_lock(&tinfo->lock, flags);
list_add_tail(&mref_a->io_head, &tinfo->mref_list);
traced_unlock(&tinfo->lock, flags);
2011-02-23 20:48:06 +00:00
wake_up_interruptible(&tinfo->event);
2010-11-26 13:45:10 +00:00
return;
done:
if (cb->cb_error < 0)
MARS_ERR("IO error %d\n", cb->cb_error);
cb->cb_fn(cb);
device_aio_ref_put(output, mref);
}
2010-12-15 12:13:18 +00:00
static int device_aio_submit(struct device_aio_output *output, struct device_aio_mref_aspect *mref_a, bool use_fdsync)
2010-11-26 13:45:10 +00:00
{
2010-12-15 12:13:18 +00:00
struct mref_object *mref = mref_a->object;
2010-11-26 13:45:10 +00:00
mm_segment_t oldfs;
int res;
struct iocb iocb = {
.aio_data = (__u64)mref_a,
2010-12-10 17:40:20 +00:00
.aio_lio_opcode = use_fdsync ? IOCB_CMD_FDSYNC : (mref->ref_rw != 0 ? IOCB_CMD_PWRITE : IOCB_CMD_PREAD),
2010-11-26 13:45:10 +00:00
.aio_fildes = output->fd,
.aio_buf = (unsigned long)mref->ref_data,
.aio_nbytes = mref->ref_len,
.aio_offset = mref->ref_pos,
};
struct iocb *iocbp = &iocb;
oldfs = get_fs();
set_fs(get_ds());
res = sys_io_submit(output->ctxp, 1, &iocbp);
set_fs(oldfs);
if (res < 0 && res != -EAGAIN)
MARS_ERR("error = %d\n", res);
return res;
}
2011-02-23 20:48:06 +00:00
static int device_aio_submit_dummy(struct device_aio_output *output)
{
mm_segment_t oldfs;
int res;
struct iocb iocb = {
};
struct iocb *iocbp = &iocb;
oldfs = get_fs();
set_fs(get_ds());
res = sys_io_submit(output->ctxp, 1, &iocbp);
set_fs(oldfs);
return res;
}
2010-11-26 13:45:10 +00:00
static int device_aio_submit_thread(void *data)
{
struct aio_threadinfo *tinfo = data;
struct device_aio_output *output = tinfo->output;
int err;
/* TODO: this is provisionary. We only need it for sys_io_submit().
* The latter should be accompanied by a future vfs_submit() or
2011-02-23 20:48:06 +00:00
* do_submit() which currently does not exist :(
2010-11-26 13:45:10 +00:00
* FIXME: corresponding cleanup NYI
*/
err = get_unused_fd();
MARS_INF("fd = %d\n", err);
if (unlikely(err < 0))
return err;
output->fd = err;
fd_install(err, output->filp);
MARS_INF("kthread has started.\n");
//set_user_nice(current, -20);
2011-02-23 20:48:06 +00:00
#if 0
fake_mm();
#else
2010-11-26 13:45:10 +00:00
MARS_INF("old mm = %p\n", current->mm);
use_mm(tinfo->mm);
MARS_INF("new mm = %p\n", current->mm);
if (!current->mm)
return 0;
2011-02-23 20:48:06 +00:00
#endif
2010-11-26 13:45:10 +00:00
while (!kthread_should_stop()) {
struct list_head *tmp = NULL;
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *mref_a;
2010-11-26 13:45:10 +00:00
unsigned long flags;
int err;
wait_event_interruptible_timeout(
tinfo->event,
!list_empty(&tinfo->mref_list) || kthread_should_stop(),
HZ);
traced_lock(&tinfo->lock, flags);
if (!list_empty(&tinfo->mref_list)) {
tmp = tinfo->mref_list.next;
list_del_init(tmp);
}
traced_unlock(&tinfo->lock, flags);
if (!tmp)
continue;
2010-12-15 12:13:18 +00:00
mref_a = container_of(tmp, struct device_aio_mref_aspect, io_head);
2010-11-26 13:45:10 +00:00
err = device_aio_submit(output, mref_a, false);
if (err == -EAGAIN) {
traced_lock(&tinfo->lock, flags);
list_add(&mref_a->io_head, &tinfo->mref_list);
traced_unlock(&tinfo->lock, flags);
msleep(10); // PROVISIONARY
continue;
}
#if 0
if (false) {
struct generic_callback *cb = mref_a->object->ref_cb;
cb->cb_error = err;
if (err < 0)
MARS_ERR("IO error %d\n", err);
cb->cb_fn(cb);
device_aio_ref_put(output, mref_a->object);
}
#endif
}
unuse_mm(tinfo->mm);
MARS_INF("kthread has stopped.\n");
2011-02-23 20:48:06 +00:00
tinfo->terminated = true;
2010-11-26 13:45:10 +00:00
return 0;
}
static int device_aio_event_thread(void *data)
{
struct aio_threadinfo *tinfo = data;
struct device_aio_output *output = tinfo->output;
struct aio_threadinfo *other = &output->tinfo[2];
MARS_INF("kthread has started.\n");
//set_user_nice(current, -20);
MARS_INF("old mm = %p\n", current->mm);
use_mm(tinfo->mm);
MARS_INF("new mm = %p\n", current->mm);
if (!current->mm)
return 0;
while (!kthread_should_stop()) {
mm_segment_t oldfs;
int count;
int bounced;
int i;
struct timespec timeout = {
2011-02-23 20:48:06 +00:00
.tv_sec = 10,
2010-11-26 13:45:10 +00:00
};
struct io_event events[MARS_MAX_AIO_READ];
oldfs = get_fs();
set_fs(get_ds());
2011-02-23 20:48:06 +00:00
/* TODO: don't timeout upon termination.
* Probably we should submit a dummy request.
*/
2010-11-26 13:45:10 +00:00
count = sys_io_getevents(output->ctxp, 1, MARS_MAX_AIO_READ, events, &timeout);
set_fs(oldfs);
//MARS_INF("count = %d\n", count);
bounced = 0;
for (i = 0; i < count; i++) {
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *mref_a = (void*)events[i].data;
2011-02-23 20:48:06 +00:00
struct mref_object *mref;
struct generic_callback *cb;
2010-11-26 13:45:10 +00:00
int err = events[i].res;
2011-02-23 20:48:06 +00:00
if (!mref_a) {
continue; // this was a dummy request
}
mref = mref_a->object;
cb = mref->ref_cb;
MARS_IO("AIO done %p pos = %lld len = %d rw = %d\n", mref, mref->ref_pos, mref->ref_len, mref->ref_rw);
2010-11-26 13:45:10 +00:00
if (output->o_fdsync
&& err >= 0
2010-12-10 17:40:20 +00:00
&& mref->ref_rw != 0
2010-11-26 13:45:10 +00:00
&& !mref_a->resubmit++) {
if (!output->filp->f_op->aio_fsync) {
unsigned long flags;
traced_lock(&other->lock, flags);
list_add(&mref_a->io_head, &other->mref_list);
traced_unlock(&other->lock, flags);
bounced++;
continue;
}
err = device_aio_submit(output, mref_a, true);
if (likely(err >= 0))
continue;
}
cb->cb_error = err;
2010-12-10 17:40:20 +00:00
if (err < 0) {
2010-11-26 13:45:10 +00:00
MARS_ERR("IO error %d\n", err);
2010-12-10 17:40:20 +00:00
} else {
2010-12-15 12:13:18 +00:00
mref->ref_flags |= MREF_UPTODATE;
2010-12-10 17:40:20 +00:00
}
2010-11-26 13:45:10 +00:00
cb->cb_fn(cb);
2010-12-10 17:40:20 +00:00
device_aio_ref_put(output, mref);
2010-11-26 13:45:10 +00:00
}
if (bounced)
2011-02-23 20:48:06 +00:00
wake_up_interruptible(&other->event);
2010-11-26 13:45:10 +00:00
}
unuse_mm(tinfo->mm);
MARS_INF("kthread has stopped.\n");
2011-02-23 20:48:06 +00:00
tinfo->terminated = true;
2010-11-26 13:45:10 +00:00
return 0;
}
/* Workaround for non-implemented aio_fsync()
*/
static int device_aio_sync_thread(void *data)
{
struct aio_threadinfo *tinfo = data;
struct device_aio_output *output = tinfo->output;
struct file *file = output->filp;
2011-02-23 20:48:06 +00:00
MARS_INF("kthread has started on '%s'.\n", output->brick->brick_name);
2010-11-26 13:45:10 +00:00
//set_user_nice(current, -20);
while (!kthread_should_stop()) {
LIST_HEAD(tmp_list);
unsigned long flags;
int err;
wait_event_interruptible_timeout(
tinfo->event,
!list_empty(&tinfo->mref_list) || kthread_should_stop(),
HZ);
traced_lock(&tinfo->lock, flags);
if (!list_empty(&tinfo->mref_list)) {
// move over the whole list
list_replace_init(&tinfo->mref_list, &tmp_list);
}
traced_unlock(&tinfo->lock, flags);
if (list_empty(&tmp_list))
continue;
err = vfs_fsync(file, file->f_path.dentry, 1);
if (err < 0)
MARS_ERR("FDSYNC error %d\n", err);
/* Signal completion for the whole list.
* No locking needed, it's on the stack.
*/
while (!list_empty(&tmp_list)) {
struct list_head *tmp = tmp_list.next;
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *mref_a = container_of(tmp, struct device_aio_mref_aspect, io_head);
2010-11-26 13:45:10 +00:00
struct generic_callback *cb = mref_a->object->ref_cb;
list_del_init(tmp);
cb->cb_error = err;
2010-12-10 17:40:20 +00:00
if (err >= 0) {
2010-12-15 12:13:18 +00:00
mref_a->object->ref_flags |= MREF_UPTODATE;
2010-12-10 17:40:20 +00:00
}
2010-11-26 13:45:10 +00:00
cb->cb_fn(cb);
device_aio_ref_put(output, mref_a->object);
}
}
MARS_INF("kthread has stopped.\n");
2011-02-23 20:48:06 +00:00
tinfo->terminated = true;
2010-11-26 13:45:10 +00:00
return 0;
}
static int device_aio_get_info(struct device_aio_output *output, struct mars_info *info)
{
struct file *file = output->filp;
2011-02-23 20:48:06 +00:00
if (unlikely(!file || !file->f_mapping || !file->f_mapping->host))
return -EINVAL;
2010-11-26 13:45:10 +00:00
info->current_size = i_size_read(file->f_mapping->host);
2011-02-23 20:48:06 +00:00
MARS_DBG("determined file size = %lld\n", info->current_size);
2010-11-26 13:45:10 +00:00
info->backing_file = file;
return 0;
}
//////////////// object / aspect constructors / destructors ///////////////
2010-12-15 12:13:18 +00:00
static int device_aio_mref_aspect_init_fn(struct generic_aspect *_ini, void *_init_data)
2010-11-26 13:45:10 +00:00
{
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *ini = (void*)_ini;
2010-11-26 13:45:10 +00:00
INIT_LIST_HEAD(&ini->io_head);
return 0;
}
2010-12-15 12:13:18 +00:00
static void device_aio_mref_aspect_exit_fn(struct generic_aspect *_ini, void *_init_data)
2010-11-26 13:45:10 +00:00
{
2010-12-15 12:13:18 +00:00
struct device_aio_mref_aspect *ini = (void*)_ini;
2010-11-26 13:45:10 +00:00
(void)ini;
}
MARS_MAKE_STATICS(device_aio);
////////////////////// brick constructors / destructors ////////////////////
static int device_aio_brick_construct(struct device_aio_brick *brick)
{
return 0;
}
2011-02-23 20:48:06 +00:00
static int device_aio_switch(struct device_aio_brick *brick)
2010-11-26 13:45:10 +00:00
{
static int index = 0;
struct device_aio_output *output = brick->outputs[0];
2011-02-23 20:48:06 +00:00
const char *path = output->output_name;
2010-11-26 13:45:10 +00:00
int flags = O_CREAT | O_RDWR | O_LARGEFILE;
int prot = 0600;
mm_segment_t oldfs;
int i;
int err = 0;
2011-02-23 20:48:06 +00:00
MARS_DBG("power.button = %d\n", brick->power.button);
if (!brick->power.button)
goto cleanup;
mars_power_led_off((void*)brick, false);
2010-11-26 13:45:10 +00:00
if (output->o_direct) {
flags |= O_DIRECT;
MARS_INF("using O_DIRECT on %s\n", path);
}
oldfs = get_fs();
set_fs(get_ds());
output->filp = filp_open(path, flags, prot);
set_fs(oldfs);
if (unlikely(IS_ERR(output->filp))) {
err = PTR_ERR(output->filp);
MARS_ERR("can't open file '%s' status=%d\n", path, err);
output->filp = NULL;
return err;
}
2011-02-23 20:48:06 +00:00
MARS_DBG("opened file '%s'\n", path);
2010-11-26 13:45:10 +00:00
if (!output->ctxp) {
2011-02-23 20:48:06 +00:00
if (!current->mm) {
MARS_ERR("mm = %p\n", current->mm);
err = -EINVAL;
goto err;
}
2010-11-26 13:45:10 +00:00
oldfs = get_fs();
set_fs(get_ds());
err = sys_io_setup(MARS_MAX_AIO, &output->ctxp);
set_fs(oldfs);
if (unlikely(err))
goto err;
}
for (i = 0; i < 3; i++) {
static int (*fn[])(void*) = {
device_aio_submit_thread,
device_aio_event_thread,
device_aio_sync_thread,
};
struct aio_threadinfo *tinfo = &output->tinfo[i];
INIT_LIST_HEAD(&tinfo->mref_list);
tinfo->output = output;
tinfo->mm = current->mm;
spin_lock_init(&tinfo->lock);
init_waitqueue_head(&tinfo->event);
2011-02-23 20:48:06 +00:00
tinfo->terminated = false;
2010-11-26 13:45:10 +00:00
tinfo->thread = kthread_create(fn[i], tinfo, "mars_aio%d", index++);
if (IS_ERR(tinfo->thread)) {
err = PTR_ERR(tinfo->thread);
MARS_ERR("cannot create thread\n");
tinfo->thread = NULL;
goto err;
}
wake_up_process(tinfo->thread);
}
MARS_INF("opened file '%s'\n", path);
2011-02-23 20:48:06 +00:00
mars_power_led_on((void*)brick, true);
MARS_DBG("successfully switched on.\n");
2010-11-26 13:45:10 +00:00
return 0;
err:
MARS_ERR("status = %d\n", err);
cleanup:
2011-02-23 20:48:06 +00:00
mars_power_led_on((void*)brick, false);
for (i = 0; i < 3; i++) {
2010-11-26 13:45:10 +00:00
struct aio_threadinfo *tinfo = &output->tinfo[i];
if (tinfo->thread) {
kthread_stop(tinfo->thread);
tinfo->thread = NULL;
}
}
2011-02-23 20:48:06 +00:00
device_aio_submit_dummy(output);
for (i = 0; i < 3; i++) {
struct aio_threadinfo *tinfo = &output->tinfo[i];
if (tinfo->thread) {
// wait for termination
wait_event_interruptible_timeout(
tinfo->event,
tinfo->terminated, 30 * HZ);
if (tinfo->terminated)
tinfo->thread = NULL;
}
2010-11-26 13:45:10 +00:00
}
2011-02-23 20:48:06 +00:00
mars_power_led_off((void*)brick,
(output->tinfo[0].thread == NULL &&
output->tinfo[1].thread == NULL &&
output->tinfo[2].thread == NULL));
if (brick->power.led_off) {
if (output->filp) {
filp_close(output->filp, NULL);
output->filp = NULL;
}
if (output->ctxp) {
#if 0 // FIXME this crashes
sys_io_destroy(output->ctxp);
#endif
output->ctxp = 0;
}
2010-11-26 13:45:10 +00:00
}
2011-02-23 20:48:06 +00:00
MARS_DBG("switch off status = %d\n", err);
2010-11-26 13:45:10 +00:00
return err;
}
static int device_aio_output_construct(struct device_aio_output *output)
{
return 0;
}
static int device_aio_output_destruct(struct device_aio_output *output)
{
2011-02-27 14:17:58 +00:00
return mars_power_button((void*)output->brick, false, true);
2010-11-26 13:45:10 +00:00
}
///////////////////////// static structs ////////////////////////
static struct device_aio_brick_ops device_aio_brick_ops = {
.brick_switch = device_aio_switch,
};
static struct device_aio_output_ops device_aio_output_ops = {
.make_object_layout = device_aio_make_object_layout,
2010-12-15 12:13:18 +00:00
.mref_get = device_aio_ref_get,
.mref_put = device_aio_ref_put,
.mref_io = device_aio_ref_io,
2010-11-26 13:45:10 +00:00
.mars_get_info = device_aio_get_info,
};
const struct device_aio_output_type device_aio_output_type = {
.type_name = "device_aio_output",
.output_size = sizeof(struct device_aio_output),
.master_ops = &device_aio_output_ops,
.output_construct = &device_aio_output_construct,
.output_destruct = &device_aio_output_destruct,
.aspect_types = device_aio_aspect_types,
.layout_code = {
2010-12-15 12:13:18 +00:00
[BRICK_OBJ_MREF] = LAYOUT_NONE,
2010-11-26 13:45:10 +00:00
}
};
static const struct device_aio_output_type *device_aio_output_types[] = {
&device_aio_output_type,
};
const struct device_aio_brick_type device_aio_brick_type = {
.type_name = "device_aio_brick",
.brick_size = sizeof(struct device_aio_brick),
.max_inputs = 0,
.max_outputs = 1,
.master_ops = &device_aio_brick_ops,
.default_output_types = device_aio_output_types,
.brick_construct = &device_aio_brick_construct,
};
EXPORT_SYMBOL_GPL(device_aio_brick_type);
////////////////// module init stuff /////////////////////////
static int __init init_device_aio(void)
{
MARS_INF("init_device_aio()\n");
return device_aio_register_brick_type();
}
static void __exit exit_device_aio(void)
{
MARS_INF("exit_device_aio()\n");
device_aio_unregister_brick_type();
}
MODULE_DESCRIPTION("MARS device_aio brick");
MODULE_AUTHOR("Thomas Schoebel-Theuer <tst@1und1.de>");
MODULE_LICENSE("GPL");
module_init(init_device_aio);
module_exit(exit_device_aio);