mars/kernel/mars_aio.h

// (c) 2010 Thomas Schoebel-Theuer / 1&1 Internet AG
#ifndef MARS_AIO_H
#define MARS_AIO_H

#include <linux/aio.h>
#include <linux/syscalls.h>

#define AIO_SUBMIT_MAX_LATENCY    1000 //   1 ms
#define AIO_IO_R_MAX_LATENCY     50000 //  50 ms
#define AIO_IO_W_MAX_LATENCY    150000 // 150 ms
#define AIO_SYNC_MAX_LATENCY    150000 // 150 ms

extern struct threshold aio_submit_threshold;
extern struct threshold aio_io_threshold[2];
extern struct threshold aio_sync_threshold;

//#define USE_CLEVER_SYNC // TODO: NYI (should result in better write performance)
#ifdef USE_CLEVER_SYNC

#include "lib_pairing_heap.h"

_PAIRING_HEAP_TYPEDEF(sync,);

struct q_sync {
	struct pairing_heap_sync *heap[MARS_PRIO_NR];
};

#endif

struct aio_mref_aspect {
	GENERIC_ASPECT(mref);
#ifdef USE_CLEVER_SYNC
	struct pairing_heap_sync heap_head;
#endif
	struct list_head io_head;
	unsigned long long enqueue_stamp;
	long long start_jiffies;
	int resubmit;
	int alloc_len;
	bool do_dealloc;
};

struct aio_brick {
	MARS_BRICK(aio);
	// parameters
	bool o_creat;
	bool o_direct;
	bool o_fdsync;
};

struct aio_input {
	MARS_INPUT(aio);
};

struct aio_threadinfo {
	struct list_head mref_list[MARS_PRIO_NR];
	struct aio_output *output;
	struct task_struct *thread;
	wait_queue_head_t event;
	wait_queue_head_t terminate_event;
	spinlock_t lock;
	int queued[MARS_PRIO_NR];
	atomic_t queued_sum;
	atomic_t total_enqueue_count;
	bool terminated;
};

struct aio_output {
	MARS_OUTPUT(aio);
        // private
	struct mapfree_info *mf;
	int fd; // FIXME: remove this!
	struct aio_threadinfo tinfo[3];
	loff_t old_size;
	aio_context_t ctxp;
	wait_queue_head_t fdsync_event;
	bool fdsync_active;
	// statistics
	int index;
	atomic_t total_read_count;
	atomic_t total_write_count;
	atomic_t total_alloc_count;
	atomic_t total_submit_count;
	atomic_t total_again_count;
	atomic_t total_delay_count;
	atomic_t total_msleep_count;
	atomic_t total_fdsync_count;
	atomic_t total_fdsync_wait_count;
	atomic_t total_mapfree_count;
	atomic_t read_count;
	atomic_t write_count;
	atomic_t alloc_count;
	atomic_t submit_count;
};

MARS_TYPES(aio);

#endif
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`// (c) 2010 Thomas Schoebel-Theuer / 1&1 Internet AG`
import mars-73.tgz 2011-03-07 10:27:38 +00:00			`#ifndef MARS_AIO_H`
			`#define MARS_AIO_H`
import mars-48.tgz 2010-11-26 13:45:10 +00:00
			`#include <linux/aio.h>`
			`#include <linux/syscalls.h>`

all: IO scheduling improvements, tuning Dastically boost random write performance on RAID controllers with BBUs. Writeback is only performed when there is no IO contention / starvation. The old IO contention controller was suited to workstations. Now server loads are well-controlled even when BBUs are present. 2012-10-15 14:35:36 +00:00			`#define AIO_SUBMIT_MAX_LATENCY 1000 // 1 ms`
			`#define AIO_IO_R_MAX_LATENCY 50000 // 50 ms`
			`#define AIO_IO_W_MAX_LATENCY 150000 // 150 ms`
			`#define AIO_SYNC_MAX_LATENCY 150000 // 150 ms`

			`extern struct threshold aio_submit_threshold;`
			`extern struct threshold aio_io_threshold[2];`
			`extern struct threshold aio_sync_threshold;`

import mars-114.tgz 2011-07-05 07:07:06 +00:00			`//#define USE_CLEVER_SYNC // TODO: NYI (should result in better write performance)`
import mars-113.tgz 2011-07-01 14:07:56 +00:00			`#ifdef USE_CLEVER_SYNC`

			`#include "lib_pairing_heap.h"`

			`_PAIRING_HEAP_TYPEDEF(sync,);`

			`struct q_sync {`
			`struct pairing_heap_sync *heap[MARS_PRIO_NR];`
			`};`

			`#endif`

import mars-73.tgz 2011-03-07 10:27:38 +00:00			`struct aio_mref_aspect {`
import mars-51.tgz 2010-12-15 12:13:18 +00:00			`GENERIC_ASPECT(mref);`
import mars-113.tgz 2011-07-01 14:07:56 +00:00			`#ifdef USE_CLEVER_SYNC`
			`struct pairing_heap_sync heap_head;`
			`#endif`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`struct list_head io_head;`
all: IO scheduling improvements, tuning Dastically boost random write performance on RAID controllers with BBUs. Writeback is only performed when there is no IO contention / starvation. The old IO contention controller was suited to workstations. Now server loads are well-controlled even when BBUs are present. 2012-10-15 14:35:36 +00:00			`unsigned long long enqueue_stamp;`
import mars-76.tgz 2011-03-10 11:40:06 +00:00			`long long start_jiffies;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`int resubmit;`
import mars-112.tgz 2011-06-30 13:15:52 +00:00			`int alloc_len;`
import mars-49.tgz 2010-12-10 17:40:20 +00:00			`bool do_dealloc;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`};`

import mars-73.tgz 2011-03-07 10:27:38 +00:00			`struct aio_brick {`
			`MARS_BRICK(aio);`
import mars-77.tgz 2011-03-11 13:57:54 +00:00			`// parameters`
aio: do mapfree_pages() via background thread This is crucial for performance, since invalidate_mapping_pages() may take a long time and consume a lot of CPU. Decouple it from time-critical worker threads. 2012-12-26 18:35:49 +00:00			`bool o_creat;`
import mars-77.tgz 2011-03-11 13:57:54 +00:00			`bool o_direct;`
			`bool o_fdsync;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`};`

import mars-73.tgz 2011-03-07 10:27:38 +00:00			`struct aio_input {`
			`MARS_INPUT(aio);`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`};`

			`struct aio_threadinfo {`
import mars-106.tgz 2011-05-19 11:36:00 +00:00			`struct list_head mref_list[MARS_PRIO_NR];`
import mars-73.tgz 2011-03-07 10:27:38 +00:00			`struct aio_output *output;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`struct task_struct *thread;`
			`wait_queue_head_t event;`
all: IO scheduling improvements, tuning Dastically boost random write performance on RAID controllers with BBUs. Writeback is only performed when there is no IO contention / starvation. The old IO contention controller was suited to workstations. Now server loads are well-controlled even when BBUs are present. 2012-10-15 14:35:36 +00:00			`wait_queue_head_t terminate_event;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`spinlock_t lock;`
all: IO scheduling improvements, tuning Dastically boost random write performance on RAID controllers with BBUs. Writeback is only performed when there is no IO contention / starvation. The old IO contention controller was suited to workstations. Now server loads are well-controlled even when BBUs are present. 2012-10-15 14:35:36 +00:00			`int queued[MARS_PRIO_NR];`
aio: use atomic_t for queued_sum 2012-12-19 15:19:33 +00:00			`atomic_t queued_sum;`
import mars-104.tgz 2011-05-06 10:25:52 +00:00			`atomic_t total_enqueue_count;`
all: IO scheduling improvements, tuning Dastically boost random write performance on RAID controllers with BBUs. Writeback is only performed when there is no IO contention / starvation. The old IO contention controller was suited to workstations. Now server loads are well-controlled even when BBUs are present. 2012-10-15 14:35:36 +00:00			`bool terminated;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`};`

import mars-73.tgz 2011-03-07 10:27:38 +00:00			`struct aio_output {`
			`MARS_OUTPUT(aio);`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`// private`
aio: do mapfree_pages() via background thread This is crucial for performance, since invalidate_mapping_pages() may take a long time and consume a lot of CPU. Decouple it from time-critical worker threads. 2012-12-26 18:35:49 +00:00			`struct mapfree_info *mf;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`int fd; // FIXME: remove this!`
			`struct aio_threadinfo tinfo[3];`
aio: fix race on file size The old code used mf->mf_max for correcting the file size, but that was wrong for multiple writes in flight. A really correct solution would have to remember all in-flight writes and compute their minimum IO position. Since that would be too costly, we just use the old size before any writes have started. This might be too conservative for extremely high load patterns (possible starvation problem). For now, take this and check whether we really need higher effort. 2013-06-03 12:49:21 +00:00			`loff_t old_size;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`aio_context_t ctxp;`
import mars-103.tgz 2011-04-29 09:36:10 +00:00			`wait_queue_head_t fdsync_event;`
remove superfluous volatile keywords 2012-02-01 12:47:35 +00:00			`bool fdsync_active;`
import mars-94.tgz 2011-04-12 15:31:08 +00:00			`// statistics`
all: IO scheduling improvements, tuning Dastically boost random write performance on RAID controllers with BBUs. Writeback is only performed when there is no IO contention / starvation. The old IO contention controller was suited to workstations. Now server loads are well-controlled even when BBUs are present. 2012-10-15 14:35:36 +00:00			`int index;`
import mars-94.tgz 2011-04-12 15:31:08 +00:00			`atomic_t total_read_count;`
			`atomic_t total_write_count;`
			`atomic_t total_alloc_count;`
aio: report total_submit_count 2012-12-20 10:32:47 +00:00			`atomic_t total_submit_count;`
aio: report total_again_count 2012-12-20 11:02:54 +00:00			`atomic_t total_again_count;`
import mars-103.tgz 2011-04-29 09:36:10 +00:00			`atomic_t total_delay_count;`
import mars-104.tgz 2011-05-06 10:25:52 +00:00			`atomic_t total_msleep_count;`
			`atomic_t total_fdsync_count;`
import mars-103.tgz 2011-04-29 09:36:10 +00:00			`atomic_t total_fdsync_wait_count;`
improve cache footprint of MARS 2012-02-20 09:34:42 +00:00			`atomic_t total_mapfree_count;`
import mars-94.tgz 2011-04-12 15:31:08 +00:00			`atomic_t read_count;`
			`atomic_t write_count;`
			`atomic_t alloc_count;`
aio: fix confusing name clash on "err", add submit_count 2012-12-18 07:00:27 +00:00			`atomic_t submit_count;`
import mars-48.tgz 2010-11-26 13:45:10 +00:00			`};`

import mars-73.tgz 2011-03-07 10:27:38 +00:00			`MARS_TYPES(aio);`
import mars-48.tgz 2010-11-26 13:45:10 +00:00
			`#endif`