RepoMirrors/mars
1
0
Fork 0
mirror of https://github.com/schoebel/mars synced 2025-02-16 20:16:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

all: fix logfile size propagation

Browse Source
This commit is contained in:
Thomas Schoebel-Theuer 2014-03-30 08:18:47 +02:00
parent 8d0ce5c9bd
commit 2f4696a9cc
7 changed files with 125 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
kernel/lib_mapfree.c
View File

@ -88,6 +88,7 @@ void _mapfree_put(struct mapfree_info *mf)
if (atomic_dec_and_test(&mf->mf_count)) {
MARS_DBG("closing file '%s' filp = %p\n", mf->mf_name, mf->mf_filp);
list_del_init(&mf->mf_head);
CHECK_HEAD_EMPTY(&mf->mf_dirty_anchor);
if (likely(mf->mf_filp)) {
mapfree_pages(mf, -1);
filp_close(mf->mf_filp, NULL);
@ -148,6 +149,7 @@ struct mapfree_info *mapfree_get(const char *name, int flags)
mf->mf_flags = flags;
INIT_LIST_HEAD(&mf->mf_head);
INIT_LIST_HEAD(&mf->mf_dirty_anchor);
atomic_set(&mf->mf_count, 1);
spin_lock_init(&mf->mf_lock);
mf->mf_max = -1;
@ -269,6 +271,74 @@ int mapfree_thread(void *data)
return 0;
}
////////////////// dirty IOs on the fly //////////////////
void mf_insert_dirty(struct mapfree_info *mf, struct dirty_info *di)
{
if (likely(di->dirty_mref)) {
unsigned long flags = 0;
traced_lock(&mf->mf_lock, flags);
list_del(&di->dirty_head);
list_add(&di->dirty_head, &mf->mf_dirty_anchor);
traced_unlock(&mf->mf_lock, flags);
}
}
EXPORT_SYMBOL_GPL(mf_insert_dirty);
void mf_remove_dirty(struct mapfree_info *mf, struct dirty_info *di)
{
if (!list_empty(&di->dirty_head)) {
unsigned long flags = 0;
traced_lock(&mf->mf_lock, flags);
list_del_init(&di->dirty_head);
traced_unlock(&mf->mf_lock, flags);
}
}
EXPORT_SYMBOL_GPL(mf_remove_dirty);
void mf_get_dirty(struct mapfree_info *mf, loff_t *min, loff_t *max, int min_stage, int max_stage)
{
struct list_head *tmp;
unsigned long flags = 0;
traced_lock(&mf->mf_lock, flags);
for (tmp = mf->mf_dirty_anchor.next; tmp != &mf->mf_dirty_anchor; tmp = tmp->next) {
struct dirty_info *di = container_of(tmp, struct dirty_info, dirty_head);
struct mref_object *mref = di->dirty_mref;
if (unlikely(!mref)) {
continue;
}
if (di->dirty_stage < min_stage || di->dirty_stage > max_stage) {
continue;
}
if (mref->ref_pos < *min) {
*min = mref->ref_pos;
}
if (mref->ref_pos + mref->ref_len > *max) {
*max = mref->ref_pos + mref->ref_len;
}
}
traced_unlock(&mf->mf_lock, flags);
}
EXPORT_SYMBOL_GPL(mf_get_dirty);
void mf_get_any_dirty(const char *filename, loff_t *min, loff_t *max, int min_stage, int max_stage)
{
struct list_head *tmp;
down_read(&mapfree_mutex);
for (tmp = mapfree_list.next; tmp != &mapfree_list; tmp = tmp->next) {
struct mapfree_info *mf = container_of(tmp, struct mapfree_info, mf_head);
if (!strcmp(mf->mf_name, filename)) {
mf_get_dirty(mf, min, max, min_stage, max_stage);
}
}
up_read(&mapfree_mutex);
}
EXPORT_SYMBOL_GPL(mf_get_any_dirty);
////////////////// module init stuff /////////////////////////
static

14
kernel/lib_mapfree.h
View File

@ -25,6 +25,7 @@ extern int mapfree_grace_keep_mb;
struct mapfree_info {
struct list_head mf_head;
struct list_head mf_dirty_anchor;
char *mf_name;
struct file *mf_filp;
int mf_flags;
@ -36,12 +37,25 @@ struct mapfree_info {
long long mf_jiffies;
};
struct dirty_info {
struct list_head dirty_head;
struct mref_object *dirty_mref;
int dirty_stage;
};
struct mapfree_info *mapfree_get(const char *filename, int flags);
void mapfree_put(struct mapfree_info *mf);
void mapfree_set(struct mapfree_info *mf, loff_t min, loff_t max);
////////////////// dirty IOs on the fly //////////////////
void mf_insert_dirty(struct mapfree_info *mf, struct dirty_info *di);
void mf_remove_dirty(struct mapfree_info *mf, struct dirty_info *di);
void mf_get_dirty(struct mapfree_info *mf, loff_t *min, loff_t *max, int min_stage, int max_stage);
void mf_get_any_dirty(const char *filename, loff_t *min, loff_t *max, int min_stage, int max_stage);
////////////////// module init stuff /////////////////////////
int __init init_mars_mapfree(void);

64
kernel/mars_aio.c
View File

@ -131,51 +131,6 @@ done:
return mref_a;
}
////////////////// dirty IOs on the fly //////////////////
static inline
void insert_dirty(struct aio_output *output, struct aio_mref_aspect *mref_a)
{
unsigned long flags = 0;
traced_lock(&output->dirty_lock, flags);
list_del(&mref_a->dirty_head);
list_add(&mref_a->dirty_head, &output->dirty_anchor);
traced_unlock(&output->dirty_lock, flags);
}
static inline
void remove_dirty(struct aio_output *output, struct aio_mref_aspect *mref_a)
{
if (!list_empty(&mref_a->dirty_head)) {
unsigned long flags = 0;
traced_lock(&output->dirty_lock, flags);
list_del_init(&mref_a->dirty_head);
traced_unlock(&output->dirty_lock, flags);
}
}
static inline
void get_dirty(struct aio_output *output, loff_t *min, loff_t *max)
{
struct list_head *tmp;
unsigned long flags = 0;
traced_lock(&output->dirty_lock, flags);
for (tmp = output->dirty_anchor.next; tmp != &output->dirty_anchor; tmp = tmp->next) {
struct aio_mref_aspect *mref_a = container_of(tmp, struct aio_mref_aspect, dirty_head);
struct mref_object *mref = mref_a->object;
if (mref->ref_pos < *min) {
*min = mref->ref_pos;
}
if (mref->ref_pos + mref->ref_len > *max) {
*max = mref->ref_pos + mref->ref_len;
}
}
traced_unlock(&output->dirty_lock, flags);
}
////////////////// own brick / input / output operations //////////////////
static
@ -211,7 +166,7 @@ loff_t get_total_size(struct aio_output *output)
*/
if (!output->brick->is_static_device) {
loff_t max = 0;
get_dirty(output, &min, &max);
mf_get_dirty(output->mf, &min, &max, 0, 99);
}
return min;
@ -320,7 +275,7 @@ done:
atomic_dec(&output->read_count);
}
remove_dirty(output, mref_a);
mf_remove_dirty(output->mf, &mref_a->di);
aio_ref_put(output, mref);
atomic_dec(&mars_global_io_flying);
@ -355,6 +310,7 @@ void _complete_all(struct list_head *tmp_list, struct aio_output *output, int er
struct list_head *tmp = tmp_list->next;
struct aio_mref_aspect *mref_a = container_of(tmp, struct aio_mref_aspect, io_head);
list_del_init(tmp);
mref_a->di.dirty_stage = 3;
_complete(output, mref_a, err);
}
}
@ -672,6 +628,7 @@ static int aio_event_thread(void *data)
if (!mref_a) {
continue; // this was a dummy request
}
mref_a->di.dirty_stage = 2;
mref = mref_a->object;
MARS_IO("AIO done %p pos = %lld len = %d rw = %d\n", mref, mref->ref_pos, mref->ref_len, mref->ref_rw);
@ -697,6 +654,7 @@ static int aio_event_thread(void *data)
continue;
}
mref_a->di.dirty_stage = 3;
_complete(output, mref_a, err);
}
@ -873,8 +831,9 @@ static int aio_submit_thread(void *data)
mapfree_set(output->mf, mref->ref_pos, -1);
mref_a->di.dirty_stage = 0;
if (mref->ref_rw) {
insert_dirty(output, mref_a);
mf_insert_dirty(output->mf, &mref_a->di);
}
mref->ref_total_size = get_total_size(output);
@ -910,6 +869,7 @@ static int aio_submit_thread(void *data)
status = aio_submit(output, mref_a, false);
if (likely(status != -EAGAIN)) {
mref_a->di.dirty_stage = 1;
break;
}
atomic_inc(&output->total_delay_count);
@ -1052,14 +1012,15 @@ static int aio_mref_aspect_init_fn(struct generic_aspect *_ini)
{
struct aio_mref_aspect *ini = (void*)_ini;
INIT_LIST_HEAD(&ini->io_head);
INIT_LIST_HEAD(&ini->dirty_head);
INIT_LIST_HEAD(&ini->di.dirty_head);
ini->di.dirty_mref = ini->object;
return 0;
}
static void aio_mref_aspect_exit_fn(struct generic_aspect *_ini)
{
struct aio_mref_aspect *ini = (void*)_ini;
CHECK_HEAD_EMPTY(&ini->dirty_head);
CHECK_HEAD_EMPTY(&ini->di.dirty_head);
CHECK_HEAD_EMPTY(&ini->io_head);
}
@ -1156,8 +1117,6 @@ cleanup:
static int aio_output_construct(struct aio_output *output)
{
INIT_LIST_HEAD(&output->dirty_anchor);
spin_lock_init(&output->dirty_lock);
init_waitqueue_head(&output->fdsync_event);
output->fd = -1;
return 0;
@ -1165,7 +1124,6 @@ static int aio_output_construct(struct aio_output *output)
static int aio_output_destruct(struct aio_output *output)
{
CHECK_HEAD_EMPTY(&output->dirty_anchor);
if (unlikely(output->fd >= 0)) {
MARS_ERR("active fd = %d detected\n", output->fd);
}

6
kernel/mars_aio.h
View File

@ -5,6 +5,8 @@
#include <linux/aio.h>
#include <linux/syscalls.h>
#include "lib_mapfree.h"
#define AIO_SUBMIT_MAX_LATENCY 1000 // 1 ms
#define AIO_IO_R_MAX_LATENCY 50000 // 50 ms
#define AIO_IO_W_MAX_LATENCY 150000 // 150 ms
@ -24,7 +26,7 @@ extern int aio_sync_mode;
struct aio_mref_aspect {
GENERIC_ASPECT(mref);
struct list_head io_head;
struct list_head dirty_head;
struct dirty_info di;
unsigned long long enqueue_stamp;
long long start_jiffies;
int resubmit;
@ -61,8 +63,6 @@ struct aio_threadinfo {
struct aio_output {
MARS_OUTPUT(aio);
// private
struct list_head dirty_anchor;
spinlock_t dirty_lock;
struct mapfree_info *mf;
int fd; // FIXME: remove this!
struct aio_threadinfo tinfo[3];

6
kernel/sy_old/mars_light.c
View File

@ -1686,6 +1686,12 @@ int check_logfile(const char *peer, struct mars_dent *remote_dent, struct mars_d
struct copy_brick *fetch_brick;
int status = 0;
// correct the remote size when necessary
if (remote_dent->d_corr_B > 0 && remote_dent->d_corr_B < src_size) {
MARS_DBG("logfile '%s' correcting src_size from %lld to %lld\n", remote_dent->d_path, src_size, remote_dent->d_corr_B);
src_size = remote_dent->d_corr_B;
}
// plausibility checks
if (unlikely(dst_size > src_size)) {
MARS_WRN("my local copy is larger than the remote one, ignoring\n");

2
kernel/sy_old/strategy.h
View File

@ -44,6 +44,8 @@ extern char *my_id(void);
char *d_rest; /* some "meaningful" rest of d_name*/ \
char *d_path; /* full absolute path */ \
struct say_channel *d_say_channel; /* for messages */ \
loff_t d_corr_A; /* logical size correction */ \
loff_t d_corr_B; /* logical size correction */ \
int d_depth; \
unsigned int d_type; /* from readdir() => often DT_UNKNOWN => don't rely on it, use new_stat.mode instead */ \
int d_class; /* for pre-grouping order */ \

19
kernel/sy_old/sy_generic.c
View File

@ -16,6 +16,7 @@
#include "strategy.h"
#include "../lib_mapfree.h"
#include "../mars_client.h"
#include <linux/syscalls.h>
@ -48,6 +49,8 @@ const struct meta mars_dent_meta[] = {
META_INI(d_type, struct mars_dent, FIELD_INT),
META_INI(d_class, struct mars_dent, FIELD_INT),
META_INI(d_serial, struct mars_dent, FIELD_INT),
META_INI(d_corr_A, struct mars_dent, FIELD_INT),
META_INI(d_corr_B, struct mars_dent, FIELD_INT),
META_INI_SUB(new_stat,struct mars_dent, mars_kstat_meta),
META_INI_SUB(old_stat,struct mars_dent, mars_kstat_meta),
META_INI(new_link, struct mars_dent, FIELD_STRING),
@ -588,6 +591,22 @@ int get_inode(char *newpath, struct mars_dent *dent)
dent->new_link = link;
}
path_put(&path);
} else if (S_ISREG(dent->new_stat.mode) && dent->d_name && !strncmp(dent->d_name, "log-", 4)) {
loff_t min = dent->new_stat.size;
loff_t max = 0;
dent->d_corr_A = 0;
dent->d_corr_B = 0;
mf_get_any_dirty(newpath, &min, &max, 0, 2);
if (min < dent->new_stat.size) {
MARS_DBG("file '%s' A size=%lld min=%lld max=%lld\n", newpath, dent->new_stat.size, min, max);
dent->d_corr_A = min;
}
mf_get_any_dirty(newpath, &min, &max, 0, 3);
if (min < dent->new_stat.size) {
MARS_DBG("file '%s' B size=%lld min=%lld max=%lld\n", newpath, dent->new_stat.size, min, max);
dent->d_corr_B = min;
}
}
if (dent->new_link)