/*
 * MARS Long Distance Replication Software
 *
 * This file is part of MARS project: http://schoebel.github.io/mars/
 *
 * Copyright (C) 2010-2014 Thomas Schoebel-Theuer
 * Copyright (C) 2011-2014 1&1 Internet AG
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */


/* Interface to a Linux device.
 * 1 Input, 0 Outputs.
 */

//#define BRICK_DEBUGGING
//#define MARS_DEBUGGING
//#define IO_DEBUGGING

#define REQUEST_MERGING
//#define ALWAYS_UNPLUG false // FIXME: does not work! single requests left over!
#define ALWAYS_UNPLUG true
#define ALWAYS_UNPLUG_FROM_EXTERNAL true
#define PREFETCH_LEN PAGE_SIZE
//#define FRONT_MERGE // FIXME: this does not work.
//#define MODIFY_READAHEAD // don't use it, otherwise sequential IO will suffer

// low-level device parameters
#define USE_MAX_SECTORS         (MARS_MAX_SEGMENT_SIZE >> 9)
#define USE_MAX_PHYS_SEGMENTS   (MARS_MAX_SEGMENT_SIZE >> 9)
#define USE_MAX_SEGMENT_SIZE    MARS_MAX_SEGMENT_SIZE
#define USE_LOGICAL_BLOCK_SIZE  512
#define USE_MAX_HW_SECTORS      1
#define USE_SEGMENT_BOUNDARY    (PAGE_SIZE-1)

#define USE_CONGESTED_FN
//#define DENY_READA

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>

#include <linux/bio.h>
#include <linux/major.h>
#include <linux/genhd.h>
#include <linux/blkdev.h>

#include "mars.h"
#include "mars_if.h"

//      remove_this

/* TBD: improve this */
#ifdef NEED_BIO_SPLIT
#undef  USE_MAX_PHYS_SEGMENTS
#define USE_MAX_PHYS_SEGMENTS   1
#endif

//      end_remove_this
///////////////////////// global tuning ////////////////////////

int if_nr_requests = 1024;

int if_throttle_start_size = 0; // in kb
EXPORT_SYMBOL_GPL(if_throttle_start_size);

struct mars_limiter if_throttle = {
	.lim_max_amount_rate = 10000,
};

///////////////////////// own type definitions ////////////////////////

#define IF_HASH_MAX   (PAGE_SIZE / sizeof(struct if_hash_anchor))
#define IF_HASH_CHUNK (PAGE_SIZE * 32)

struct if_hash_anchor {
	spinlock_t hash_lock;
	struct list_head hash_anchor;
};

///////////////////////// own static definitions ////////////////////////

// TODO: check bounds, ensure that free minor numbers are recycled
static int device_minor = 0;

//////////////// object / aspect constructors / destructors ///////////////

///////////////////////// linux operations ////////////////////////

#ifdef MARS_HAS_GENERIC_BLK_ACCOUNTING

/* Disbaled by default, for saving overhead */
int mars_io_acct = 0;

static
void _if_start_io_acct(struct if_input *input, struct bio_wrapper *biow)
{
	struct bio *bio;

	if (!mars_io_acct)
		return;

	bio = biow->bio;
	biow->start_time = jiffies;
	generic_start_io_acct(
#ifdef MARS_HAS_NEW_GENERIC_BLK_ACCOUNTING
			      input->q,
#endif
			      bio_data_dir(bio),
			      bio->bi_iter.bi_size >> 9,
			      &input->disk->part0);
}

static
void _if_end_io_acct(struct if_input *input, struct bio_wrapper *biow)
{
	struct bio *bio;

	if (!biow->start_time)
		return;

	bio = biow->bio;
	generic_end_io_acct(
#ifdef MARS_HAS_NEW_GENERIC_BLK_ACCOUNTING
			    input->q,
#endif
			    bio_data_dir(bio),
			    &input->disk->part0,
			    biow->start_time);
}

#elif defined(MARS_HAS_OLD_BLK_ACCOUNTING)

static
void _if_start_io_acct(struct if_input *input, struct bio_wrapper *biow)
{
	struct bio *bio = biow->bio;
	const int rw = bio_data_dir(bio);
	const int cpu = part_stat_lock();

	(void)cpu;
	part_round_stats(cpu, &input->disk->part0);
	part_stat_inc(cpu, &input->disk->part0, ios[rw]);
//      remove_this
#ifdef MARS_HAS_BVEC_ITER
//      end_remove_this
	part_stat_add(cpu, &input->disk->part0, sectors[rw], bio->bi_iter.bi_size >> 9);
//      remove_this
#else
	part_stat_add(cpu, &input->disk->part0, sectors[rw], bio->bi_size >> 9);
#endif
//      end_remove_this
	part_inc_in_flight(&input->disk->part0, rw);
	part_stat_unlock();
	biow->start_time = jiffies;
}

static
void _if_end_io_acct(struct if_input *input, struct bio_wrapper *biow)
{
	unsigned long duration = jiffies - biow->start_time;
	struct bio *bio = biow->bio;
	const int rw = bio_data_dir(bio);
	const int cpu = part_stat_lock();
	(void)cpu;
	part_stat_add(cpu, &input->disk->part0, ticks[rw], duration);
	part_round_stats(cpu, &input->disk->part0);
	part_dec_in_flight(&input->disk->part0, rw);
	part_stat_unlock();
}

#else // MARS_HAS_OLD_BLK_ACCOUNTING

#define _if_start_io_acct(...) do {} while (0)
#define _if_end_io_acct(...)   do {} while (0)

#endif

/* Abstract from interface changes
 */
static
void _call_bio_endio(struct if_brick *brick, struct bio *bio, int error)
{
#ifdef MARS_HAS_BI_STATUS
	bio->bi_status = errno_to_blk_status(error);
	bio_endio(bio);
#else
#ifdef MARS_HAS_BI_ERROR
	bio->bi_error = error;
	bio_endio(bio);
#else
	bio_endio(bio, error);
#endif
#endif
	/* Just a hint for userspace, no strictness needed */
	brick->completion_stamp = get_real_lamport();
}

/* callback
 */
static
void if_endio(struct generic_callback *cb)
{
	struct if_mref_aspect *mref_a = cb->cb_private;
	struct if_input *input;
	struct if_brick *brick;
	int k;
	int error;

	LAST_CALLBACK(cb);
	if (unlikely(!mref_a || !mref_a->object)) {
		MARS_FAT("mref_a = %p mref = %p, something is very wrong here!\n", mref_a, mref_a->object);
		return;
	}
	input = mref_a->input;
	CHECK_PTR(input, err);

	mars_trace(mref_a->object, "if_endio");
	mars_log_trace(mref_a->object);

	MARS_IO("flags = %ux bio_count = %d\n", mref->ref_flags, mref_a->bio_count);

	for (k = 0; k < mref_a->bio_count; k++) {
		struct bio_wrapper *biow;
		struct bio *bio;

		biow = mref_a->orig_biow[k];
		mref_a->orig_biow[k] = NULL;
		CHECK_PTR(biow, err);

		CHECK_ATOMIC(&biow->bi_comp_cnt, 1);
		if (!atomic_dec_and_test(&biow->bi_comp_cnt)) {
			continue;
		}

		bio = biow->bio;
		CHECK_PTR_NULL(bio, err);

		_if_end_io_acct(input, biow);

		error = CALLBACK_ERROR(mref_a->object);
		if (unlikely(error < 0)) {
			if (likely(input->brick))
				input->brick->error_code = error;
		} else { // bio conventions are slightly different...
			error = 0;
//      remove_this
#ifdef MARS_HAS_BVEC_ITER
//      end_remove_this
			bio->bi_iter.bi_size = 0;
//      remove_this
#else
			bio->bi_size = 0;
#endif
//      end_remove_this
		}
		MARS_IO("calling end_io() flags = %ux error = %d\n",
			mref->ref_flags, error);
		_call_bio_endio(input->brick, bio, error);
		brick_mem_free(biow);
	}
	brick = input->brick;
	if (mref_a->object->ref_flags & MREF_WRITE) {
		atomic_dec(&brick->write_flying_count);
	} else {
		atomic_dec(&brick->read_flying_count);
	}
#ifdef IO_DEBUGGING
		{
			char *txt = brick->ops->brick_statistics(brick, false);
			MARS_IO("%s", txt);
			brick_string_free(txt);
		}
#endif
	MARS_IO("finished.\n");
	return;

err:
	MARS_FAT("error in callback, giving up\n");
}

/* Kick off plugged mrefs
 */
static
void _if_unplug(struct if_input *input)
{
	struct if_brick *brick = input->brick;
	LIST_HEAD(tmp_list);
	unsigned long flags;

#ifdef CONFIG_MARS_DEBUG
	might_sleep();
#endif

	MARS_IO("plugged_count = %d\n", atomic_read(&input->plugged_count));

	down(&input->kick_sem);
	traced_lock(&input->req_lock, flags);
#ifdef USE_TIMER
	del_timer(&input->timer);
#endif
	if (!list_empty(&input->plug_anchor)) {
		// move over the whole list
		list_replace_init(&input->plug_anchor, &tmp_list);
		atomic_set(&input->plugged_count, 0);
	}
  	traced_unlock(&input->req_lock, flags);
	up(&input->kick_sem);

	while (!list_empty(&tmp_list)) {
		struct if_mref_aspect *mref_a;
		struct mref_object *mref;
		int hash_index;
		unsigned long flags;

		mref_a = container_of(tmp_list.next, struct if_mref_aspect, plug_head);
		list_del_init(&mref_a->plug_head);

		hash_index = mref_a->hash_index;
		traced_lock(&input->hash_table[hash_index].hash_lock, flags);
		list_del_init(&mref_a->hash_head);
		traced_unlock(&input->hash_table[hash_index].hash_lock, flags);

                mref = mref_a->object;

		if (unlikely(mref_a->current_len > mref_a->max_len)) {
			MARS_ERR("request len %d > %d\n", mref_a->current_len, mref_a->max_len);
		}
		mref->ref_len = mref_a->current_len;

		mars_trace(mref, "if_unplug");

#ifdef CONFIG_MARS_DEBUG
		atomic_inc(&input->total_fire_count);
#endif
		brick = input->brick;
		if (mref->ref_flags & MREF_WRITE) {
			atomic_inc(&brick->write_flying_count);
		} else {
			atomic_inc(&brick->read_flying_count);
		}
#ifdef CONFIG_MARS_DEBUG
		if (mref->ref_flags & MREF_SKIP_SYNC)
			atomic_inc(&input->total_skip_sync_count);
#endif

		GENERIC_INPUT_CALL(input, mref_io, mref);
		GENERIC_INPUT_CALL(input, mref_put, mref);
	}
#ifdef IO_DEBUGGING
	{
		struct if_brick *brick = input->brick;
		char *txt = brick->ops->brick_statistics(brick, false);
		MARS_IO("%s", txt);
		brick_string_free(txt);
	}
#endif
}

#ifndef BLK_MAX_REQUEST_COUNT
#ifdef USE_TIMER
static
void if_timer(unsigned long data)
{
	MARS_IO("\n");
	_if_unplug((void*)data);
}
#endif
#endif // BLK_MAX_REQUEST_COUNT

/* accept a linux bio, convert to mref and call buf_io() on it.
 */
static
/* see dece16353ef47d8d33f5302bc158072a9d65e26f */
#ifdef BLK_QC_T_NONE
#ifdef NEED_BIO_SPLIT
//      end_remove_this
blk_qc_t _if_make_request(struct request_queue *q, struct bio *bio)
//      remove_this
#else
blk_qc_t if_make_request(struct request_queue *q, struct bio *bio)
#endif
#elif defined(BIO_CPU_AFFINE)
int if_make_request(struct request_queue *q, struct bio *bio)
#else
#ifdef NEED_BIO_SPLIT
void _if_make_request(struct request_queue *q, struct bio *bio)
#else
void if_make_request(struct request_queue *q, struct bio *bio)
#endif
#endif
{
	struct if_input *input = q->queuedata;
	struct if_brick *brick = input->brick;

	/* Original flags of the source bio
	 */
	const int  rw      = bio_data_dir(bio);
	const int  sectors = bio_sectors(bio);
//      remove_this
/* Adapt to different kernel versions.
 * This is a maintainance nightmare.
 * Most is provisionary.
 */
#if defined(BIO_RW_RQ_MASK) || defined(BIO_FLUSH)
	const bool ahead   = bio_rw_flagged(bio, BIO_RW_AHEAD) && rw == READ;
	const bool barrier = bio_rw_flagged(bio, BIO_RW_BARRIER);
	const bool syncio  = bio_rw_flagged(bio, BIO_RW_SYNCIO);
	const bool unplug  = bio_rw_flagged(bio, BIO_RW_UNPLUG);
	const bool meta    = bio_rw_flagged(bio, BIO_RW_META);
	const bool discard = bio_rw_flagged(bio, BIO_RW_DISCARD);
	const bool noidle  = bio_rw_flagged(bio, BIO_RW_NOIDLE);
#elif defined(REQ_FLUSH) && defined(REQ_SYNC)
#define _flagged(x) (bio->bi_rw & (x))
	const bool ahead   = _flagged(REQ_RAHEAD) && rw == READ;
	const bool barrier = _flagged(REQ_FLUSH);
	const bool syncio  = _flagged(REQ_SYNC);
	const bool unplug  = false;
	const bool meta    = _flagged(REQ_META);
	const bool discard = _flagged(REQ_DISCARD);
	const bool noidle  = _flagged(REQ_THROTTLED);
#elif defined(MARS_HAS_NEW_BIO_OP)
//      end_remove_this
#define _flagged(x) (bio->bi_opf & (x))
	const bool ahead   = _flagged(REQ_RAHEAD) && rw == READ;
	const bool barrier = false;
	const bool syncio  = _flagged(REQ_SYNC);
	const bool unplug  = false;
	const bool meta    = _flagged(REQ_META);
	const bool discard = bio_op(bio) == REQ_OP_DISCARD;
#ifdef MARS_HAS_BIO_THROTTLED
	const bool noidle  = !bio_flagged(bio, BIO_THROTTLED);
#else
	const bool noidle  = _flagged(REQ_THROTTLED);
#endif
//      remove_this
#else
#error Cannot decode the bio flags
#endif
//      end_remove_this
	const int  prio    = bio_prio(bio);

	/* Transform into MARS flags
	 */
	const int  ref_prio =
		(prio == IOPRIO_CLASS_RT || (meta | syncio)) ?
		MARS_PRIO_HIGH :
		(prio == IOPRIO_CLASS_IDLE) ?
		MARS_PRIO_LOW :
		MARS_PRIO_NORMAL;
	const bool do_unplug = ALWAYS_UNPLUG | unplug | noidle;
	const bool do_skip_sync = brick->skip_sync && !(barrier | syncio);

	struct bio_wrapper *biow;
	struct mref_object *mref = NULL;
	struct if_mref_aspect *mref_a;
//      remove_this
#ifdef MARS_HAS_BVEC_ITER
//      end_remove_this
	struct bio_vec bvec;
	struct bvec_iter i;
	loff_t pos = ((loff_t)bio->bi_iter.bi_sector) << 9; // TODO: make dynamic
	int total_len = bio->bi_iter.bi_size;
//      remove_this
#else
	struct bio_vec *bvec;
	int i;
	loff_t pos = ((loff_t)bio->bi_sector) << 9; // TODO: make dynamic
	int total_len = bio->bi_size;
#endif
//      end_remove_this
	int kb = (total_len + 512) / 1024;
	bool assigned = false;
        int error = -EINVAL;

	bind_to_channel(brick->say_channel, current);

	MARS_IO("bio %p "
		"size = %d "
		"rw = %d "
		"sectors = %d "
		"ahead = %d "
		"barrier = %d "
		"syncio = %d "
		"unplug = %d "
		"meta = %d "
		"discard = %d "
		"noidle = %d "
		"prio = %d "
		"pos = %lldd "
		"total_len = %d\n",
		bio,
		bio->bi_size,
		rw,
		sectors,
		ahead,
		barrier,
		syncio,
		unplug,
		meta,
		discard,
		noidle,
		prio,
		pos,
		total_len);

	might_sleep();

	if (unlikely(!sectors)) {
		_if_unplug(input);
		/* THINK: usually this happens only at write barriers.
		 * We have no "barrier" operation in MARS, since
		 * callback semantics should always denote
		 * "writethrough accomplished".
		 * In case of exceptional semantics, we need to do
		 * something here. For now, we do just nothing.
		 */
		error = 0;
		_call_bio_endio(brick, bio, 0);
		goto done;
	}

	// throttling of too big write requests
	if (rw && if_throttle_start_size > 0) {
		if (kb >= if_throttle_start_size)
			mars_limit_sleep(&if_throttle, kb);
	}

	/* gather statistics on IOPS etc */
	mars_limit(&brick->io_limiter, kb);

#ifdef DENY_READA // provisinary -- we should introduce an equivalent of READA also to the MARS infrastructure
	if (ahead) {
#ifdef CONFIG_MARS_DEBUG
		atomic_inc(&input->total_reada_count);
#endif
		_call_bio_endio(brick, bio, -EWOULDBLOCK);
		error = 0;
		goto done;
	}
#else
	(void)ahead; // shut up gcc
#endif
	if (unlikely(discard)) { // NYI
		error = 0;
		_call_bio_endio(brick, bio, 0);
		goto done;
	}

	biow = brick_mem_alloc(sizeof(struct bio_wrapper));
	CHECK_PTR(biow, err);
	biow->bio = bio;
	atomic_set(&biow->bi_comp_cnt, 0);

#ifdef CONFIG_MARS_DEBUG
	if (rw) {
		atomic_inc(&input->total_write_count);
	} else {
		atomic_inc(&input->total_read_count);
	}
#endif

	_if_start_io_acct(input, biow);

	/* FIXME: THIS IS PROVISIONARY (use event instead)
	 */
	while (unlikely(!brick->power.led_on)) {
		brick_msleep(100);
	}

	down(&input->kick_sem);

	bio_for_each_segment(bvec, bio, i) {
//      remove_this
#ifdef MARS_HAS_BVEC_ITER
//      end_remove_this
		struct page *page = bvec.bv_page;
		int bv_len = bvec.bv_len;
		int offset = bvec.bv_offset;
//      remove_this
#else
		struct page *page = bvec->bv_page;
		int bv_len = bvec->bv_len;
		int offset = bvec->bv_offset;
#endif
//      end_remove_this
		void *data;

#ifdef ARCH_HAS_KMAP
#error FIXME: the current infrastructure cannot deal with HIGHMEM / kmap()
#endif
		data = page_address(page);
		MARS_IO("page = %p data = %p\n", page, data);
		error = -EINVAL;
		if (unlikely(!data))
			break;

		data += offset;

		while (bv_len > 0) {
			struct list_head *tmp;
			int hash_index;
			int this_len = 0;
			unsigned long flags;

			mref = NULL;
			mref_a = NULL;

			MARS_IO("rw = %d i = %d pos = %lld  bv_page = %p bv_offset = %d data = %p bv_len = %d\n", rw, i, pos, bvec->bv_page, bvec->bv_offset, data, bv_len);

			hash_index = (pos / IF_HASH_CHUNK) % IF_HASH_MAX;

#ifdef REQUEST_MERGING
			traced_lock(&input->hash_table[hash_index].hash_lock, flags);
			for (tmp = input->hash_table[hash_index].hash_anchor.next; tmp != &input->hash_table[hash_index].hash_anchor; tmp = tmp->next) {
				struct if_mref_aspect *tmp_a;
				struct mref_object *tmp_mref;
				int i;

				tmp_a = container_of(tmp, struct if_mref_aspect, hash_head);
				tmp_mref = tmp_a->object;
				if (tmp_a->orig_page != page ||
				    (tmp_mref->ref_flags & MREF_WRITE) != (rw ? MREF_WRITE : 0) ||
				    tmp_a->bio_count >= MAX_BIO ||
				    tmp_a->current_len + bv_len > tmp_a->max_len) {
					continue;
				}

				if (tmp_mref->ref_data + tmp_a->current_len == data) {
					goto merge_end;
#ifdef FRONT_MERGE // FIXME: this cannot work. ref_data must never be changed. pre-allocate from offset 0 instead.
				} else if (data + bv_len == tmp_mref->ref_data) {
					goto merge_front;
#endif
				}
				continue;

#ifdef FRONT_MERGE // FIXME: this cannot work. ref_data must never be changed. pre-allocate from offset 0 instead.
			merge_front:
				tmp_mref->ref_data = data;
#endif
			merge_end:
				tmp_a->current_len += bv_len;
				mref = tmp_mref;
				mref_a = tmp_a;
				this_len = bv_len;
				if (!do_skip_sync) {
					mref->ref_flags &= ~MREF_SKIP_SYNC;
				}

				for (i = 0; i < mref_a->bio_count; i++) {
					if (mref_a->orig_biow[i]->bio == bio) {
						goto unlock;
					}
				}

				CHECK_ATOMIC(&biow->bi_comp_cnt, 0);
				atomic_inc(&biow->bi_comp_cnt);
				mref_a->orig_biow[mref_a->bio_count++] = biow;
				assigned = true;
				goto unlock;
			} // foreach hash collision list member

		unlock:
			traced_unlock(&input->hash_table[hash_index].hash_lock, flags);
#endif
			if (!mref) {
				int prefetch_len;
				error = -ENOMEM;
				mref = if_alloc_mref(brick);
				if (unlikely(!mref)) {
					up(&input->kick_sem);
					goto err;
				}
				mref_a = if_mref_get_aspect(brick, mref);
				if (unlikely(!mref_a)) {
					up(&input->kick_sem);
					goto err;
				}

#ifdef PREFETCH_LEN
				prefetch_len = PREFETCH_LEN - offset;
#if 1
				// TODO: this restriction is too strong to be useful for performance boosts. Do better.
				if (prefetch_len > total_len) {
					prefetch_len = total_len;
				}
#endif
				if (pos + prefetch_len > brick->dev_size) {
					prefetch_len = brick->dev_size - pos;
				}
				if (prefetch_len < bv_len) {
					prefetch_len = bv_len;
				}
#else
				prefetch_len = bv_len;
#endif

				SETUP_CALLBACK(mref, if_endio, mref_a);

				mref_a->input = input;
				mref->ref_flags = rw ? MREF_WRITE | MREF_MAY_WRITE : 0;
				mref->ref_pos = pos;
				mref->ref_len = prefetch_len;
				mref->ref_data = data; // direct IO
				mref->ref_prio = ref_prio;
				mref_a->orig_page = page;

				error = GENERIC_INPUT_CALL(input, mref_get, mref);
				if (unlikely(error < 0)) {
					up(&input->kick_sem);
					goto err;
				}
				
				mars_trace(mref, "if_start");

				this_len = mref->ref_len; // now may be shorter than originally requested.
				mref_a->max_len = this_len;
				if (this_len > bv_len) {
					this_len = bv_len;
				}
				mref_a->current_len = this_len;
#ifdef CONFIG_MARS_DEBUG
				if (rw) {
					atomic_inc(&input->total_mref_write_count);
				} else {
					atomic_inc(&input->total_mref_read_count);
				}
#endif

				CHECK_ATOMIC(&biow->bi_comp_cnt, 0);
				atomic_inc(&biow->bi_comp_cnt);
				mref_a->orig_biow[0] = biow;
				mref_a->bio_count = 1;
				assigned = true;

				/* When a bio with multiple biovecs is split into
				 * multiple mrefs, only the last one should be
				 * working in synchronous writethrough mode.
				 */
				mref->ref_flags |= MREF_SKIP_SYNC;
//      remove_this
#ifdef MARS_HAS_BVEC_ITER
//      end_remove_this
				if (!do_skip_sync && i.bi_idx + 1 >= bio->bi_iter.bi_idx) {
					mref->ref_flags &= ~MREF_SKIP_SYNC;
				}
//      remove_this
#else
				if (!do_skip_sync && i + 1 >= bio->bi_vcnt) {
					mref->ref_flags &= ~MREF_SKIP_SYNC;
				}
#endif
//      end_remove_this

				atomic_inc(&input->plugged_count);

				mref_a->hash_index = hash_index;
				traced_lock(&input->hash_table[hash_index].hash_lock, flags);
				list_add_tail(&mref_a->hash_head, &input->hash_table[hash_index].hash_anchor);
				traced_unlock(&input->hash_table[hash_index].hash_lock, flags);

				traced_lock(&input->req_lock, flags);
				list_add_tail(&mref_a->plug_head, &input->plug_anchor);
				traced_unlock(&input->req_lock, flags);
			} // !mref

			pos += this_len;
			data += this_len;
			bv_len -= this_len;
			total_len -= this_len;
		} // while bv_len > 0
	} // foreach bvec

	up(&input->kick_sem);

	if (likely(!total_len)) {
		error = 0;
	} else {
		MARS_ERR("bad rest len = %d\n", total_len);
	}

err:

#ifdef IO_DEBUGGING
	{
		char *txt = brick->ops->brick_statistics(brick, false);
		MARS_IO("%s", txt);
		brick_string_free(txt);
	}
#endif

	if (error < 0) {
		MARS_ERR("cannot submit request from bio, status=%d\n", error);
		if (!assigned) {
			brick->error_code = error;
			_call_bio_endio(brick, bio, error);
		}
	}

	if (do_unplug ||
	   (brick && brick->max_plugged > 0 && atomic_read(&input->plugged_count) > brick->max_plugged)) {
		_if_unplug(input);
	}
#ifdef USE_TIMER
	else {
		unsigned long flags;
		traced_lock(&input->req_lock, flags);
		if (timer_pending(&input->timer)) {
			del_timer(&input->timer);
		}
		input->timer.function = if_timer;
		input->timer.data = (unsigned long)input;
		input->timer.expires = jiffies + USE_TIMER;
		add_timer(&input->timer);
		traced_unlock(&input->req_lock, flags);
	}
#endif

done:
	remove_binding_from(brick->say_channel, current);

//      remove_this
/* see dece16353ef47d8d33f5302bc158072a9d65e26f */
#ifdef BLK_QC_T_NONE
//      end_remove_this
	return BLK_QC_T_NONE;
//      remove_this
#elif defined(BIO_CPU_AFFINE)
	return error;
#else
	return;
#endif
}

#ifdef NEED_BIO_SPLIT
static
#ifdef BLK_QC_T_NONE
blk_qc_t if_make_request(struct request_queue *q, struct bio *bio)
#else
void if_make_request(struct request_queue *q, struct bio *bio)
#endif
{
#ifdef MARS_HAS_BIO_SPLIT2
	blk_queue_split(q, &bio);
#else
	blk_queue_split(q, &bio, q->bio_split);
#endif
#ifdef BLK_QC_T_NONE
	return _if_make_request(q, bio);
#else
	_if_make_request(q, bio);
#endif
}

#endif

#ifndef BLK_MAX_REQUEST_COUNT
//static
void if_unplug(struct request_queue *q)
{
	struct if_input *input = q->queuedata;
	int was_plugged = 1;
#if 1
	spin_lock_irq(q->queue_lock);
	was_plugged = blk_remove_plug(q);
	spin_unlock_irq(q->queue_lock);
#else
	queue_flag_clear_unlocked(QUEUE_FLAG_PLUGGED, q);
#endif

	was_plugged += atomic_read(&input->plugged_count);

	MARS_IO("block layer called UNPLUG was_plugged = %d\n", was_plugged);
	if (ALWAYS_UNPLUG_FROM_EXTERNAL || was_plugged) {
		_if_unplug(input);
	}
}
#endif

//static
int mars_congested(void *data, int bdi_bits)
{
	struct if_input *input = data;
	struct if_brick *brick = input->brick;
	int ret = 0;

#ifdef WB_STAT_BATCH /* changed by 4452226ea276e74fc3e252c88d9bb7e8f8e44bf0 */
	if (bdi_bits & (1 << WB_sync_congested) &&
	    atomic_read(&brick->read_flying_count) > 0) {
		ret |= (1 << WB_sync_congested);
	}
	if (bdi_bits & (1 << WB_async_congested) &&
	    atomic_read(&brick->write_flying_count) > 0) {
		ret |= (1 << WB_async_congested);
	}
#else /* old code */
	if (bdi_bits & (1 << BDI_sync_congested) &&
	    atomic_read(&brick->read_flying_count) > 0) {
		ret |= (1 << BDI_sync_congested);
	}
	if (bdi_bits & (1 << BDI_async_congested) &&
	    atomic_read(&brick->write_flying_count) > 0) {
		ret |= (1 << BDI_async_congested);
	}
#endif
	return ret;
}

//      remove_this
#ifdef MARS_HAS_MERGE_BVEC
static
int mars_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec)
{
	unsigned int bio_size = bvm->bi_size;
	if (!bio_size) {
		return bvec->bv_len;
	}
	return 128;
}
#endif

//      end_remove_this
static
loff_t if_get_capacity(struct if_brick *brick)
{
	/* Don't read always, read only when unknown.
	 * brick->dev_size may be different from underlying sizes,
	 * e.g. when the size symlink indicates a logically smaller
	 * device than physically.
	 */
	if (brick->dev_size <= 0) {
		struct if_input *input = brick->inputs[0];
		int status;

		status = GENERIC_INPUT_CALL(input, mars_get_info, &brick->info);
		if (unlikely(status < 0)) {
			MARS_ERR("cannot get device info, status=%d\n", status);
			return 0;
		}
		MARS_INF("determined default capacity: %lld bytes\n", brick->info.current_size);
		brick->dev_size = brick->info.current_size;
	}
	return brick->dev_size;
}

static
void if_set_capacity(struct if_input *input, loff_t capacity)
{
	CHECK_PTR(input->disk, done);
	CHECK_PTR(input->disk->disk_name, done);
	MARS_INF("new capacity of '%s': %lld bytes\n", input->disk->disk_name, capacity);
	input->capacity = capacity;
	set_capacity(input->disk, capacity >> 9);
	if (likely(input->bdev && input->bdev->bd_inode)) {
		i_size_write(input->bdev->bd_inode, capacity);
	}
done:;
}

static const struct block_device_operations if_blkdev_ops;

static
int check_io_done(struct if_brick *brick, bool do_wait)
{
	for (;;) {
		int nr = atomic_read(&brick->read_flying_count) +
			atomic_read(&brick->write_flying_count);

		if (nr <= 0 || !do_wait)
			return nr;
		MARS_INF("%d IO requests not yet completed\n", nr);
		brick_msleep(1000 / HZ + 1);
	}
}

static int if_switch(struct if_brick *brick)
{
	struct if_input *input = brick->inputs[0];
	struct request_queue *q;
	struct gendisk *disk;
	int minor;
	int status = 0;

	down(&brick->switch_sem);

	// brick is in operation
	if (brick->power.button && brick->power.led_on) {
		loff_t capacity;
		capacity = if_get_capacity(brick);
		if (capacity > 0 && capacity != input->capacity) {
			MARS_INF("changing capacity from %lld to %lld\n", (long long)input->capacity, (long long)capacity);
			if_set_capacity(input, capacity);
		}
	}

	// brick should be switched on
	if (brick->power.button && brick->power.led_off) {
		loff_t capacity;
#ifdef MARS_HAS_BDI_GET
		struct backing_dev_info *bdi;
#endif

		mars_power_led_off((void*)brick,  false);
		brick->say_channel = get_binding(current);

		status = -ENOMEM;
		q = blk_alloc_queue(GFP_MARS);
		if (!q) {
			MARS_ERR("cannot allocate device request queue\n");
			goto is_down;
		}
		q->queuedata = input;
		input->q = q;
		
		disk = alloc_disk(1);
		if (!disk) {
			MARS_ERR("cannot allocate gendisk\n");
			goto is_down;
		}

		minor = device_minor++; //TODO: protect against races (e.g. atomic_t)
		set_disk_ro(disk, true);

		disk->queue = q;
		disk->major = MARS_MAJOR; //TODO: make this dynamic for >256 devices
		disk->first_minor = minor;
		disk->fops = &if_blkdev_ops;
		snprintf(disk->disk_name, sizeof(disk->disk_name),  "mars/%s", brick->brick_name);
		disk->private_data = input;
		input->disk = disk;
		capacity = if_get_capacity(brick);
		MARS_DBG("created device name %s, capacity=%lld\n", disk->disk_name, capacity);
		if_set_capacity(input, capacity);
		
		blk_queue_make_request(q, if_make_request);
#ifdef USE_MAX_SECTORS
#ifdef MAX_SEGMENT_SIZE
		MARS_DBG("blk_queue_max_sectors()\n");
		blk_queue_max_sectors(q, USE_MAX_SECTORS);
#else
		MARS_DBG("blk_queue_max_hw_sectors()\n");
		blk_queue_max_hw_sectors(q, USE_MAX_SECTORS);
#endif
#endif
#ifdef USE_MAX_PHYS_SEGMENTS
#ifdef MAX_SEGMENT_SIZE
		MARS_DBG("blk_queue_max_phys_segments()\n");
		blk_queue_max_phys_segments(q, USE_MAX_PHYS_SEGMENTS);
#else
		MARS_DBG("blk_queue_max_segments()\n");
		blk_queue_max_segments(q, USE_MAX_PHYS_SEGMENTS);
#endif
#endif
#ifdef USE_MAX_HW_SEGMENTS
		MARS_DBG("blk_queue_max_hw_segments()\n");
		blk_queue_max_hw_segments(q, USE_MAX_HW_SEGMENTS);
#endif
#ifdef USE_MAX_HW_SECTORS
		MARS_DBG("blk_queue_max_hw_sectors()\n");
		blk_queue_max_hw_sectors(q, USE_MAX_HW_SECTORS);
#endif
#ifdef USE_MAX_SEGMENT_SIZE
		MARS_DBG("blk_queue_max_segment_size()\n");
		blk_queue_max_segment_size(q, USE_MAX_SEGMENT_SIZE);
#endif
#ifdef USE_LOGICAL_BLOCK_SIZE
		if (brick->info.tf_align >= 512) {
			MARS_DBG("blk_queue_physical_block_size(%d)\n", brick->info.tf_align);
			blk_queue_physical_block_size(q, brick->info.tf_align);
		}
		if (brick->info.tf_min_size >= 512) {
			MARS_DBG("blk_queue_logical_block_size(%d)\n", brick->info.tf_min_size);
			blk_queue_logical_block_size(q, brick->info.tf_min_size);
		} else {
			MARS_DBG("blk_queue_logical_block_size()\n");
			blk_queue_logical_block_size(q, USE_LOGICAL_BLOCK_SIZE);
		}
#endif
#ifdef USE_SEGMENT_BOUNDARY
		MARS_DBG("blk_queue_segment_boundary()\n");
		blk_queue_segment_boundary(q, USE_SEGMENT_BOUNDARY);
#endif
#ifdef QUEUE_ORDERED_DRAIN
		MARS_DBG("blk_queue_ordered()\n");
		blk_queue_ordered(q, QUEUE_ORDERED_DRAIN, NULL);
#endif
		MARS_DBG("blk_queue_bounce_limit()\n");
		blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
#ifndef BLK_MAX_REQUEST_COUNT
		MARS_DBG("unplug_fn\n");
		q->unplug_fn = if_unplug;
#endif
		MARS_DBG("queue_lock\n");
		q->queue_lock = &input->req_lock; // needed!
		
		input->bdev = bdget(MKDEV(disk->major, minor));
		/* we have no partitions. we contain only ourselves. */
		input->bdev->bd_contains = input->bdev;

#ifdef MARS_HAS_BDI_GET
		bdi = input->bdev->bd_bdi;
#ifdef MODIFY_READAHEAD
		MARS_INF("ra_pages OLD = %lu NEW = %d\n",
			 bdi->ra_pages, brick->readahead);
		bdi->ra_pages = brick->readahead;
#endif
#ifdef USE_CONGESTED_FN
		MARS_DBG("congested_fn\n");
		bdi->congested_fn = mars_congested;
		bdi->congested_data = input;
#endif

#else

#ifdef MODIFY_READAHEAD
		MARS_INF("ra_pages OLD = %lu NEW = %d\n", q->backing_dev_info.ra_pages, brick->readahead);
		q->backing_dev_info.ra_pages = brick->readahead;
#endif
#ifdef USE_CONGESTED_FN
		MARS_DBG("congested_fn\n");
		q->backing_dev_info.congested_fn = mars_congested;
		q->backing_dev_info.congested_data = input;
#endif
#endif
//      remove_this
#ifdef MARS_HAS_MERGE_BVEC
		MARS_DBG("blk_queue_merge_bvec()\n");
		blk_queue_merge_bvec(q, mars_merge_bvec);
#endif

//      end_remove_this
		q->nr_requests = if_nr_requests;
		// point of no return
		MARS_DBG("add_disk()\n");
		add_disk(disk);
#if 1
		set_disk_ro(disk, false);
#else
		set_device_ro(input->bdev, 0); // TODO: implement modes
#endif

		// report success
		mars_power_led_on((void*)brick, true);
		status = 0;
	}

	// brick should be switched off
	if (!brick->power.button && !brick->power.led_off) {
		int opened;
		int plugged;
		int flying;

		mars_power_led_on((void*)brick, false);
		disk = input->disk;
		if (!disk)
			goto is_down;

		opened = atomic_read(&brick->open_count);
		if (unlikely(opened > 0)) {
			MARS_INF("device '%s' is open %d times, cannot shutdown\n", disk->disk_name, opened);
			status = -EBUSY;
			goto done; // don't indicate "off" status
		}
		plugged = atomic_read(&input->plugged_count);
		if (unlikely(plugged > 0)) {
			MARS_INF("device '%s' has %d plugged requests, cannot shutdown\n", disk->disk_name, plugged);
			status = -EBUSY;
			goto done; // don't indicate "off" status
		}
		flying = check_io_done(brick, false);
		if (unlikely(flying > 0)) {
			MARS_INF("device '%s' has %d flying requests, cannot shutdown\n", disk->disk_name, flying);
			status = -EBUSY;
			goto done; // don't indicate "off" status
		}
		MARS_DBG("calling del_gendisk()\n");
		del_gendisk(input->disk);
		/* There might be subtle races */
		check_io_done(brick, true);
		if (input->bdev) {
#if defined(USE_CONGESTED_FN) && defined(MARS_HAS_BDI_GET)
			struct backing_dev_info *bdi;

			/* ensure that callbacks will stop */
			MARS_DBG("unregister congested_fn\n");
			bdi = input->bdev->bd_bdi;
			if (bdi)
				bdi->congested_fn = NULL;
#endif
			MARS_DBG("calling bdput()\n");
			bdput(input->bdev);
			input->bdev = NULL;
		}
		MARS_DBG("calling put_disk()\n");
		put_disk(input->disk);
		input->disk = NULL;
		q = input->q;
		if (q) {
#if defined(USE_CONGESTED_FN) && !defined(MARS_HAS_BDI_GET)
			/* ensure that callbacks will stop */
			MARS_DBG("unregister congested_fn\n");
			q->backing_dev_info.congested_fn = NULL;
#endif
			blk_cleanup_queue(q);
			input->q = NULL;
		}
		status = 0;
	is_down:
		mars_power_led_off((void*)brick, true);
	}

done:
	up(&brick->switch_sem);
	return status;
}

//////////////// interface to the outer world (kernel) ///////////////

static int if_open(struct block_device *bdev, fmode_t mode)
{
	struct if_input *input;
	struct if_brick *brick;

	if (unlikely(!bdev || !bdev->bd_disk)) {
		MARS_ERR("----------------------- INVAL ------------------------------\n");
		return -EINVAL;
	}

	input = bdev->bd_disk->private_data;

	if (unlikely(!input || !input->brick)) {
		MARS_ERR("----------------------- BAD IF SETUP ------------------------------\n");
		return -EINVAL;
	}
	brick = input->brick;

	down(&brick->switch_sem);

	if (unlikely(!brick->power.led_on)) {
		MARS_INF("----------------------- BUSY %d ------------------------------\n", atomic_read(&brick->open_count));
		up(&brick->switch_sem);
		return -EBUSY;
	}

	atomic_inc(&brick->open_count);

	MARS_INF("----------------------- OPEN %d ------------------------------\n", atomic_read(&brick->open_count));
	mars_remote_trigger(MARS_TRIGGER_LOCAL | MARS_TRIGGER_TO_REMOTE);

	up(&brick->switch_sem);
	return 0;
}

static
//      remove_this
#ifdef MARS_HAS_VOID_RELEASE
//      end_remove_this
void
//      remove_this
#else
int
#endif
//      end_remove_this
if_release(struct gendisk *gd, fmode_t mode)
{
	struct if_input *input = gd->private_data;
	struct if_brick *brick = input->brick;
	int nr;

	nr = atomic_read(&brick->open_count);
	MARS_INF("----------------------- CLOSE %d ------------------------------\n",
		 nr);
	if (nr <= 1)
		check_io_done(brick, true);

	if (atomic_dec_and_test(&brick->open_count)) {
		check_io_done(brick, true);

		MARS_DBG("status button=%d led_on=%d led_off=%d\n", brick->power.button, brick->power.led_on, brick->power.led_off);
		brick->error_code = 0;
		mars_remote_trigger(MARS_TRIGGER_LOCAL | MARS_TRIGGER_TO_REMOTE);
	}
//      remove_this
#ifndef MARS_HAS_VOID_RELEASE
	return 0;
#endif
//      end_remove_this
}

static const struct block_device_operations if_blkdev_ops = {
	.owner =   THIS_MODULE,
	.open =    if_open,
	.release = if_release,

};

//////////////// informational / statistics ///////////////

static
char *if_statistics(struct if_brick *brick, int verbose)
{
	struct if_input *input = brick->inputs[0];
	char *res = brick_string_alloc(512);
#ifdef CONFIG_MARS_DEBUG
	int tmp0 = atomic_read(&input->total_reada_count); 
	int tmp1 = atomic_read(&input->total_read_count); 
	int tmp2 = atomic_read(&input->total_mref_read_count);
	int tmp3 = atomic_read(&input->total_write_count); 
	int tmp4 = atomic_read(&input->total_mref_write_count);
#endif
	if (!res)
		return NULL;
	snprintf(res, 512,
		 "disk = %p "
#ifdef CONFIG_MARS_DEBUG
		 "total reada = %d "
		 "reads = %d "
		 "mref_reads = %d (%d%%) "
		 "writes = %d "
		 "mref_writes = %d (%d%%) "
		 "empty = %d "
		 "fired = %d "
		 "skip_sync = %d "
#endif
		 "| "
		 "opened = %d "
		 "plugged = %d "
		 "flying reads = %d "
		 "flying writes = %d\n",
		 input->disk,
#ifdef CONFIG_MARS_DEBUG
		 tmp0,
		 tmp1,
		 tmp2,
		 tmp1 ? tmp2 * 100 / tmp1 : 0,
		 tmp3,
		 tmp4,
		 tmp3 ? tmp4 * 100 / tmp3 : 0,
		 atomic_read(&input->total_empty_count),
		 atomic_read(&input->total_fire_count),
		 atomic_read(&input->total_skip_sync_count),
#endif
		 atomic_read(&brick->open_count),
		 atomic_read(&input->plugged_count),
		 atomic_read(&brick->read_flying_count),
		 atomic_read(&brick->write_flying_count));
	return res;
}

static
void if_reset_statistics(struct if_brick *brick)
{
#ifdef CONFIG_MARS_DEBUG
	struct if_input *input = brick->inputs[0];

	atomic_set(&input->total_read_count, 0);
	atomic_set(&input->total_write_count, 0);
	atomic_set(&input->total_empty_count, 0);
	atomic_set(&input->total_fire_count, 0);
	atomic_set(&input->total_skip_sync_count, 0);
	atomic_set(&input->total_mref_read_count, 0);
	atomic_set(&input->total_mref_write_count, 0);
#endif
}

////////////////// own brick / input / output operations //////////////////

// none

//////////////// object / aspect constructors / destructors ///////////////

static int if_mref_aspect_init_fn(struct generic_aspect *_ini)
{
	struct if_mref_aspect *ini = (void*)_ini;
	INIT_LIST_HEAD(&ini->plug_head);
	INIT_LIST_HEAD(&ini->hash_head);
	return 0;
}

static void if_mref_aspect_exit_fn(struct generic_aspect *_ini)
{
	struct if_mref_aspect *ini = (void*)_ini;
	CHECK_HEAD_EMPTY(&ini->plug_head);
	CHECK_HEAD_EMPTY(&ini->hash_head);
}

MARS_MAKE_STATICS(if);

//////////////////////// constructors / destructors ////////////////////////

static int if_brick_construct(struct if_brick *brick)
{
	sema_init(&brick->switch_sem, 1);
	atomic_set(&brick->open_count, 0);
	atomic_set(&brick->read_flying_count, 0);
	atomic_set(&brick->write_flying_count, 0);
	return 0;
}

static int if_brick_destruct(struct if_brick *brick)
{
	return 0;
}

static int if_input_construct(struct if_input *input)
{
	int i;

	input->hash_table = brick_block_alloc(0, PAGE_SIZE);
	if (unlikely(!input->hash_table)) {
		MARS_ERR("cannot allocate hash table\n");
		return -ENOMEM;
	}
	for (i = 0; i < IF_HASH_MAX; i++) {
		spin_lock_init(&input->hash_table[i].hash_lock);
		INIT_LIST_HEAD(&input->hash_table[i].hash_anchor);
	}
	INIT_LIST_HEAD(&input->plug_anchor);
	sema_init(&input->kick_sem, 1);
	spin_lock_init(&input->req_lock);
	atomic_set(&input->plugged_count, 0);
#ifdef USE_TIMER
	init_timer(&input->timer);
#endif
	return 0;
}

static int if_input_destruct(struct if_input *input)
{
	int i;

	for (i = 0; i < IF_HASH_MAX; i++) {
		CHECK_HEAD_EMPTY(&input->hash_table[i].hash_anchor);
	}
	CHECK_HEAD_EMPTY(&input->plug_anchor);
	brick_block_free(input->hash_table, PAGE_SIZE);
	return 0;
}

static int if_output_construct(struct if_output *output)
{
	return 0;
}

///////////////////////// static structs ////////////////////////

static struct if_brick_ops if_brick_ops = {
	.brick_switch = if_switch,
	.brick_statistics = if_statistics,
	.reset_statistics = if_reset_statistics,
};

static struct if_output_ops if_output_ops = {
};

const struct if_input_type if_input_type = {
	.type_name = "if_input",
	.input_size = sizeof(struct if_input),
	.input_construct = &if_input_construct,
	.input_destruct = &if_input_destruct,
};

static const struct if_input_type *if_input_types[] = {
	&if_input_type,
};

const struct if_output_type if_output_type = {
	.type_name = "if_output",
	.output_size = sizeof(struct if_output),
	.master_ops = &if_output_ops,
	.output_construct = &if_output_construct,
};

static const struct if_output_type *if_output_types[] = {
	&if_output_type,
};


const struct if_brick_type if_brick_type = {
	.type_name = "if_brick",
	.brick_size = sizeof(struct if_brick),
	.max_inputs = 1,
	.max_outputs = 0,
	.master_ops = &if_brick_ops,
	.aspect_types = if_aspect_types,
	.default_input_types = if_input_types,
	.default_output_types = if_output_types,
	.brick_construct = &if_brick_construct,
	.brick_destruct = &if_brick_destruct,
};
EXPORT_SYMBOL_GPL(if_brick_type);

////////////////// module init stuff /////////////////////////

void exit_mars_if(void)
{
	int status;
	MARS_INF("exit_if()\n");
	status = if_unregister_brick_type();
	unregister_blkdev(MARS_MAJOR, "mars");
}

int __init init_mars_if(void)
{
	int status;

	(void)if_aspect_types; // not used, shut up gcc

	MARS_INF("init_if()\n");
	status = register_blkdev(MARS_MAJOR, "mars");
	if (status)
		return status;
	status = if_register_brick_type();
	if (status)
		goto err_device;
	return status;
err_device:
	MARS_ERR("init_if() status=%d\n", status);
	exit_mars_if();
	return status;
}