EOL: fully merge branch 'mars0.1.y' into mars0.1a.y

ChangeLog

@@ -30,14 +30,11 @@ Example: mars0.3alpha*:
 
 Release Conventions / Branches / Tagnames
 -----------------------------------------
-	FLOW OF BUGFIXES: 0.1 -> 0.1a -> ...
 
-	mars0.1 series (stable, will go EOL soon):
-		 - Will run in parallel to branch 0.1a for a few
-		   months, and then go EOL.
+	mars0.1 series (now EOL):
 		 - Unstable tagnames: light0.1beta%d.%d (obsolete)
-		 - Stable branch: mars0.1.y
-		 - Stable tagnames: mars0.1stable%02d
+		 - Stable branch: mars0.1.y (obsolete)
+		 - Stable tagnames: mars0.1stable%02d (obsolete)
 
 	mars0.1a series (stable):
 		 New master branch. Now stable.
@@ -339,9 +336,10 @@ mars0.1balpha0
 -----------------------------------
 Changelog for series 0.1:
 
-Attention! This branch will go EOL around March 2019.
-	Branch mars0.1a.y is now the new master branch, and all
-	commits from the 0.1 series will be fully merged.
+Attention! This branch is now EOL.
+	Everything has been merged into branch mars0.1a.y which
+	is also the master branch.
+	PLEASE UPGRADE to the new branch.
 	Upgrade is easy: just rollout the new marsadm version,
 	install the new kernel modules, and load them where possible.
 	Mixed operation of different versions is no problem,
@@ -352,8 +350,13 @@ Attention! This branch will go EOL around March 2019.
 	Motivation: branch 0.1a is productive for several years at 1&1.
 	Experiences: now runs provably better than 0.1.y with
 	better performance, smoother, etc.
-	And even more stable, although the 0.1a releases were
-	called "beta" up to now.
+
+mars0.1stable74 (last stable release in branch mars0.1.y)
+	* Major fix, only relevant for a corner case:
+	  Writeback made no human-visible progress under
+	  multiple weird preconditions.
+	* Minor usability improvement: marsadm view shows
+	  more fancy details on logfile numbers.
 
 mars0.1stable73
 	* Critical fix, only relevant for kernels >= 4.2.x:

kernel/mars_trans_logger.c

@@ -97,6 +97,11 @@ int trans_logger_do_crc =
 int trans_logger_mem_usage; // in KB
 
 int trans_logger_pressure_limit = 0;
+int trans_logger_disable_pressure = 0; /* only for testing */
+
+int trans_logger_report_interval = 2; /* seconds */
+
+int trans_logger_writeback_maxage = 900; /* seconds */
 
 int trans_logger_max_interleave = -1;
 
@@ -594,7 +599,7 @@ void _inf_callback(struct trans_logger_input *input, bool force)
 {
 	if (!force &&
 	    input->inf_last_jiffies &&
-	    input->inf_last_jiffies + 4 * HZ > (long long)jiffies)
+	    input->inf_last_jiffies + trans_logger_report_interval * HZ > (long long)jiffies)
 		return;
 	
 	if (input->inf.inf_callback && input->is_operating) {
@@ -603,6 +608,10 @@ void _inf_callback(struct trans_logger_input *input, bool force)
 		input->inf.inf_callback(&input->inf);
 
 		input->inf_last_jiffies = jiffies;
+		if (input->inf_min_old != input->inf.inf_min_pos) {
+			input->inf_min_jiffies = input->inf_last_jiffies;
+			input->inf_min_old = input->inf.inf_min_pos;
+		}
 	} else {
 		MARS_DBG("%p skipped callback, callback = %p is_operating = %d\n", input, input->inf.inf_callback, input->is_operating);
 	}
@@ -2313,7 +2322,28 @@ struct rank_info global_rank_mref_flying[] = {
 static inline
 bool _check_pressure(struct trans_logger_brick *brick)
 {
-	int active =
+	int active;
+
+	/*
+	 * Writeback IO starvation can occur when too many concurrents reads
+	 * are filling the IO channnels for a very long time (e.g. when
+	 * full backups are running in parallel to a highly loaded server).
+	 * Ensure a very minimum writeback speed, depending on real time
+	 * (as expected by humans).
+	 */
+	if (trans_logger_writeback_maxage) {
+		struct trans_logger_input *log_input;
+		long long inf_min_jiffies = 0;
+
+		log_input = brick->inputs[brick->log_input_nr];
+		if (log_input)
+			inf_min_jiffies = log_input->inf_min_jiffies;
+		if (!inf_min_jiffies ||
+		    trans_logger_writeback_maxage * HZ + inf_min_jiffies <= (long long)jiffies)
+			return false;
+	}
+
+	active =
 		atomic_read(&brick->any_fly_count) +
 		brick->q_phase[0].q_queued + brick->q_phase[0].q_active;
 
@@ -2358,6 +2388,11 @@ int _do_ranking(struct trans_logger_brick *brick)
 
 		MARS_IO("local_mem_used = %d\n", local_mem_used);
 	}
+	/* disable pressure ONLY for testing */
+	if (trans_logger_disable_pressure > 0)
+		pressure_mode = 0;
+	else if (trans_logger_disable_pressure < 0)
+		pressure_mode = 1;
 	if (delay_callers) {
 		if (!brick->delay_callers) {
 			brick->delay_callers = true;

kernel/mars_trans_logger.h

@@ -51,6 +51,9 @@ extern int trans_logger_completion_semantics;
 extern int trans_logger_do_crc;
 extern int trans_logger_mem_usage; // in KB
 extern int trans_logger_pressure_limit;
+extern int trans_logger_disable_pressure; /* only for testing */
+extern int trans_logger_report_interval;
+extern int trans_logger_writeback_maxage;
 extern int trans_logger_max_interleave;
 extern int trans_logger_resume;
 extern int trans_logger_replay_timeout; // in s
@@ -280,6 +283,8 @@ struct trans_logger_input {
 	// private
 	struct log_status logst;
 	struct list_head pos_list;
+	long long inf_min_jiffies;
+	long long inf_min_old;
 	long long inf_last_jiffies;
 	struct semaphore inf_mutex;
 };

kernel/sy_old/mars_main.c

@@ -658,7 +658,7 @@ static LIST_HEAD(rot_anchor);
 // TUNING
 
 int mars_mem_percent = 20;
-EXPORT_SYMBOL_GPL(mars_mem_percent);
+int mars_mem_gb = 16;
 
 #define CONF_TRANS_SHADOW_LIMIT (1024 * 128) // don't fill the hashtable too much
 
@@ -5935,12 +5935,18 @@ static int _main_thread(void *data)
 			mars_mem_percent = 0;
 		if (mars_mem_percent > 70)
 			mars_mem_percent = 70;
+		if (mars_mem_gb < 1)
+			mars_mem_gb = 1;
 		memlimit = (long long)brick_global_memavail * mars_mem_percent / 100;
+		if (memlimit > (long long)mars_mem_gb * 1024 * 1024)
+			memlimit = (long long)mars_mem_gb * 1024 * 1024;
+
 		/* Dynamic memlimit when /mars is becoming full */
 		if (memlimit > global_remaining_space / 4)
 			memlimit = global_remaining_space / 4;
 		if (memlimit < 4)
 			memlimit = 4;
+
 		brick_global_memlimit = memlimit;
 
 		_global_sync_nr = 0;

kernel/sy_old/mars_proc.c

@@ -355,10 +355,14 @@ struct ctl_table mars_table[] = {
 	INT_ENTRY("mapfree_period_sec",   mapfree_period_sec,     0600),
 	INT_ENTRY("mapfree_grace_keep_mb", mapfree_grace_keep_mb, 0600),
 	INT_ENTRY("logger_pressure_limit", trans_logger_pressure_limit, 0600),
+	INT_ENTRY("logger_disable_pressure", trans_logger_disable_pressure, 0600),
+	INT_ENTRY("logger_writeback_maxage_s", trans_logger_writeback_maxage, 0600),
+	INT_ENTRY("logger_report_interval_s", trans_logger_report_interval, 0600),
 	INT_ENTRY("logger_max_interleave", trans_logger_max_interleave, 0600),
 	INT_ENTRY("logger_resume",        trans_logger_resume,    0600),
 	INT_ENTRY("logger_replay_timeout_sec", trans_logger_replay_timeout, 0600),
 	INT_ENTRY("mem_limit_percent",    mars_mem_percent,       0600),
+	INT_ENTRY("mem_limit_gb",         mars_mem_gb,            0600),
 	INT_ENTRY("logger_mem_used_kb",   trans_logger_mem_usage, 0400),
 	INT_ENTRY("mem_used_raw_kb",      brick_global_block_used,0400),
 #ifdef CONFIG_MARS_MEM_PREALLOC

kernel/sy_old/strategy.h

@@ -125,7 +125,6 @@ typedef int (*mars_dent_worker_fn)(struct mars_global *global, struct mars_dent
 extern int mars_dent_work(struct mars_global *global, char *dirname, int allocsize, mars_dent_checker_fn checker, mars_dent_worker_fn worker, void *buf, int maxdepth);
 extern struct mars_dent *_mars_find_dent(struct mars_global *global, const char *path);
 extern struct mars_dent *mars_find_dent(struct mars_global *global, const char *path);
-extern int mars_find_dent_all(struct mars_global *global, char *prefix, struct mars_dent ***table);
 extern void mars_kill_dent(struct mars_dent *dent);
 extern void mars_free_dent(struct mars_dent *dent);
 extern void mars_free_dent_all(struct mars_global *global, struct list_head *anchor);
@@ -225,6 +224,7 @@ extern int mars_max_loadavg;
 #endif
 
 extern int mars_mem_percent;
+extern int mars_mem_gb;
 
 extern int main_checker(struct mars_dent *parent, const char *_name, int namlen, unsigned int d_type, int *prefix, int *serial, bool *use_channel);
 

kernel/sy_old/sy_generic.c

@@ -1668,40 +1668,6 @@ struct mars_dent *mars_find_dent(struct mars_global *global, const char *path)
 }
 EXPORT_SYMBOL_GPL(mars_find_dent);
 
-int mars_find_dent_all(struct mars_global *global, char *prefix, struct mars_dent ***table)
-{
-	int max = 1024; // provisionary
-	int count = 0;
-	struct list_head *tmp;
-	struct mars_dent **res = brick_zmem_alloc(max * sizeof(void*));
-	int prefix_len = strlen(prefix);
-
-	*table = res;
-	if (unlikely(!res || !global))
-		goto done;
-
-	down_read(&global->dent_mutex);
-	for (tmp = global->dent_anchor.next; tmp != &global->dent_anchor; tmp = tmp->next) {
-		struct mars_dent *tmp_dent = container_of(tmp, struct mars_dent, dent_link);
-		int this_len;
-		if (!tmp_dent->d_path) {
-			continue;
-		}
-		this_len = strlen(tmp_dent->d_path);
-		if (this_len < prefix_len || strncmp(tmp_dent->d_path, prefix, prefix_len)) {
-			continue;
-		}
-		res[count++] = tmp_dent;
-		if (count >= max)
-			break;
-	}
-	up_read(&global->dent_mutex);
-
-done:
-	return count;
-}
-EXPORT_SYMBOL_GPL(mars_find_dent_all);
-
 void mars_kill_dent(struct mars_dent *dent)
 {
 	dent->d_killme = true;

userspace/marsadm

@@ -5254,14 +5254,18 @@ my %complex_macros =
 
    "replinfo"
    => "%let{amount}{%human-numbers{}{ }{ }{%replay-pos{}}{%fetch-size{}}}"
-   .  "%sprintf{     replaying: %s  %.2f%%  (%d/%d)%s  logs: [%d..%d]\n}"
+   .  "%let{logs}{%replay-lognr{}}"
+   .  "%let{l2}{%fetch-lognr{}}"
+   .  "%let{l3}{%work-lognr{}}"
+   .  "%if{%>{%{l2}}{%{logs}}}{%append{logs}{::%{l2}}}"
+   .  "%if{%>{%{l3}}{%{l2}}}{%append{logs}{..%{l3}}}"
+   .  "%sprintf{     replaying: %s  %.2f%%  (%d/%d)%s  logs: [%s]\n}"
    .          "{%progress{20}{%replay-pos{}}{%fetch-pos{}}{%fetch-size{}}}"
    .          "{%work-percent{}}"
    .          "{%{amount}{ }{0}}"
    .          "{%{amount}{ }{1}}"
    .          "{%{amount}{ }{2}}"
-   .          "{%replay-lognr{}}"
-   .          "{%fetch-lognr{}}"
+   .          "{%{logs}}"
    .  "%call{fetch-line}"
    .  "%call{replay-line}",