mars/docu/mars-user-manual.lyx

#LyX 2.3 created this file. For more info see http://www.lyx.org/
\lyxformat 544
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass scrreprt
\begin_preamble
\usepackage{listings}
\end_preamble
\options abstracton,most,usenames,dvipsnames
\use_default_options true
\begin_modules
customHeadersFooters
enumitem
fixltx2e
tcolorbox
\end_modules
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman "default" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family rmdefault
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\use_microtype false
\use_dash_ligatures false
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize 10
\spacing single
\use_hyperref true
\pdf_title "MARS User Manual"
\pdf_author "Thomas Schöbel-Theuer"
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen true
\pdf_bookmarksopenlevel 2
\pdf_breaklinks true
\pdf_pdfborder true
\pdf_colorlinks true
\pdf_backref section
\pdf_pdfusetitle true
\papersize a4paper
\use_geometry true
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\biblio_style plain
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\justification true
\use_refstyle 1
\use_minted 0
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 3.7cm
\topmargin 2.7cm
\rightmargin 2.8cm
\bottommargin 2.3cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\is_math_indent 0
\math_numbering_side default
\quotes_style english
\dynamic_quotes 0
\papercolumns 1
\papersides 2
\paperpagestyle headings
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
title{MARS User Manual}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand input
preview true
filename "common-front-matter.lyx"

\end_inset


\end_layout

\begin_layout Chapter*
Preface
\end_layout

\begin_layout Section*
Introduction
\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand input
preview true
filename "common-introduction.lyx"

\end_inset


\end_layout

\begin_layout Section*
Purpose
\end_layout

\begin_layout Standard
This document explains how to install, setup and run a storage replication
 system on a Linux based platform using MARS.
\end_layout

\begin_layout Section*
Scope
\end_layout

\begin_layout Standard
The following topics are covered withing this document:
\end_layout

\begin_layout Itemize
preconditions: what you need.
\end_layout

\begin_layout Itemize
creating a Linux kernel module for MARS.
\end_layout

\begin_layout Itemize
creation of a MARS cluster.
\end_layout

\begin_layout Itemize
creation and operation of MARS resources.
\end_layout

\begin_layout Itemize
dynamic creation / deletion of additional replicas, and how migrate data
 this way.
\end_layout

\begin_layout Itemize
hints for monitoring.
\end_layout

\begin_layout Itemize
troubleshooting (see appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Handout-for-Midnight"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
\end_layout

\begin_layout Section*
Audience
\end_layout

\begin_layout Standard
This document is written for experienced sysadmins with working knowledge
 on the following methods and technologies:
\end_layout

\begin_layout Itemize
Setup and operation of LVM (Logical Volume Manager) under Linux.
\end_layout

\begin_layout Itemize
Operation of storage systems.
\end_layout

\begin_layout Itemize
Ability to patch and to compile and install a customized Linux kernel.
 Patches are already provided ready-to-use, so no full developer knowledge
 is needed.
\end_layout

\begin_layout Section*
How to use this document
\end_layout

\begin_layout Standard
Please start with the very short introduction 
\begin_inset CommandInset ref
LatexCommand formatted
reference "chap:Briefing:-how-MARS"
plural "false"
caps "false"
noprefix "false"

\end_inset

 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:Briefing:-how-MARS"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
If you want to install MARS, read 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:Quick-Start-Guide"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
If you just want to operate a MARS installation which is already set up,
 go on to 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:HOWTO-operation-of"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
If you already have some experiences with MARS and just need some details
 about marsadm commands, chapter 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:The-Sysadmin-Interface"
plural "false"
caps "false"
noprefix "false"

\end_inset

 is a kind of 
\begin_inset Quotes eld
\end_inset

reference
\begin_inset Quotes erd
\end_inset

 for you.
\end_layout

\begin_layout Standard
Users who want to go deeper into tuning should read 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:Tuning,-tips-and"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
Automation via interfacing to 
\family typewriter
systemd
\family default
 is described in 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:Advanced-users:-automation"
plural "false"
caps "false"
noprefix "false"

\end_inset

, as well as tips for writing your own automation scripts.
\end_layout

\begin_layout Standard
Finally, troubleshooting is explained in appendix 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:Handout-for-Midnight"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
Some non-standard expert tricks (e.g.
 for mass operation of thousands of instances) can be found in the following
 appendices.
\end_layout

\begin_layout Section*
Related documents
\end_layout

\begin_layout Itemize

\family typewriter
mars-architecture-guide.pdf
\family default
: explains usage scenarios.
\end_layout

\begin_layout Itemize

\family typewriter
football-user-manual.pdf
\family default
: for sysadmins and userspace developers who want to use Football.
\end_layout

\begin_layout Itemize

\family typewriter
mars-for-kernel-developers.pdf
\family default
: some infos for kernel developers.
\end_layout

\begin_layout Standard
\begin_inset CommandInset toc
LatexCommand tableofcontents

\end_inset


\end_layout

\begin_layout Chapter
Briefing: how MARS works
\begin_inset CommandInset label
LatexCommand label
name "chap:Briefing:-how-MARS"

\end_inset


\end_layout

\begin_layout Section
Typical MARS replication setup
\begin_inset CommandInset label
LatexCommand label
name "sec:Typical-MARS-replication"

\end_inset


\end_layout

\begin_layout Standard
Typical recommended usage is replication of multiple Logical Volumes (LVs)
 directly at bare metal (never inside of VMs), similar to DRBD:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/typical-mars-setup.fig
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
At the primary (active) side A, the applications are running.
 At each of the secondary (passive) sides B and C, only the underlying LV
 replicas are updated via the replication network traffic.
 The filesystem is not mounted at any secondary side, and the applications
 are not running there.
 However, the roles may be switched at any time, and then the application
 will for example run at Datacenter B in primary role, while the corresponding
 LV replicas will then be in secondary (passive) role at datacenters A and
 C.
\end_layout

\begin_layout Standard
An advantage of multiple LV replication is that primary and secondary roles
 can be 
\series bold
\emph on
individually
\emph default
 switched at runtime
\series default
.
 For example, if you have 10 LVs in each of your servers, 6 LVs may currently
 run in datacenter A in primary role, while the other 4 LVs are running
 in datacenter B, while datacenter C is dimensioned for less CPU power,
 and is mainly intended for additional 
\begin_inset Quotes eld
\end_inset

emergency backup
\begin_inset Quotes erd
\end_inset

 replicas.
 For instance, such a 3-datacenter configuration may be used for load balancing
 during overload peaks, or for switchover due to kernel security updates,
 and much more.
\end_layout

\begin_layout Standard
Further setups are also possible.
 For example, you might replicate physical disks.
 However, this would be less flexible because your volumes must then uniformly
 run in the 
\emph on
same
\emph default
 datacenter at the same time.
\end_layout

\begin_layout Standard
In addition, MARS' replicated block devices may be exported via iSCSI or
 other protocols.
 Filesystems residing on top of MARS may be exported via NFS or glusterfs,
 etc.
 For more details, please consult 
\family typewriter
mars-architecture-guide.pdf
\family default
.
\end_layout

\begin_layout Section
The Transaction Logger
\begin_inset CommandInset label
LatexCommand label
name "sec:The-Transaction-Logger"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/MARS_Data_Flow.pdf
	lyxscale 60
	width 100text%

\end_inset


\end_layout

\begin_layout Standard
\noindent
The basic idea of MARS is to record all changes made to your LV in a so-called
 
\series bold
transaction logfile
\series default
.
 
\emph on
Any
\emph default
 write reqeuest is treated like a transaction which changes the contents
 of your LV.
\end_layout

\begin_layout Standard
This is similar in concept to some database systems (c.f.
 MySQL replication), but there exists no separate 
\begin_inset Quotes eld
\end_inset

commit
\begin_inset Quotes erd
\end_inset

 operation: 
\emph on
any
\emph default
 write request is acting like a commit.
\end_layout

\begin_layout Standard
The picture shows the flow of write requests.
 Let's start with the primary node.
\end_layout

\begin_layout Standard
Write requests directed at the virtual block device 
\family typewriter
/dev/mars/mydata
\family default
 are first buffered in a 
\emph on
temporary
\emph default
 kernel memory buffer.
\end_layout

\begin_layout Standard
The temporary kernel memory buffer serves multiple purposes:
\end_layout

\begin_layout Itemize
It keeps track of the 
\series bold
order
\series default
 of write operations.
\end_layout

\begin_layout Itemize
Additionally, it keeps track of the positions in the underlying LV 
\family typewriter
/dev/lv-x/mydata
\family default
.
 In particular, it detects when the same block is overwritten multiple times.
\end_layout

\begin_layout Itemize
Writeback to the underlying LV may occur in a different order than submission
 order.
 On magnetic disk media, this may lead to a noticeable 
\series bold
performance boost
\series default
, as shown in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:IO-Performance-Tuning"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Itemize
During pending write operations, any concurrent reads from the same locations
 are served from the temporary memory buffer.
\end_layout

\begin_layout Standard
After the write has been buffered in the temporary memory buffer, the internal
 transaction logger kernel thread creates a so-called 
\emph on
log entry
\emph default
 and starts an 
\begin_inset Quotes eld
\end_inset

append
\begin_inset Quotes erd
\end_inset

 operation on the transaction logfile.
 The log entry contains vital information such as the logical block number
 in the underlying LV, the length of the data, a timestamp, some header
 magic in order to detect corruption, the log entry sequence number, of
 course the data itself, and optional information like an MD5 checksum or
 compression information.
\end_layout

\begin_layout Standard
Once the log entries
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that the order of log records present in the transaction log defines
 a total order among the write requests which is 
\emph on
compatible
\emph default
 to the partial order of write requests issued on 
\family typewriter
/dev/mars/mydata
\family default
.
\end_layout

\begin_layout Plain Layout
Also notice that despite its sequential nature, the transaction logfile
 is typically 
\emph on
not
\emph default
 the performance bottleneck of the system.
 At least on magnetic media, appending to a logfile is almost purely sequential
 IO, it runs much faster than random IO.
\end_layout

\end_inset

 have been written through to the 
\family typewriter
/mars/
\family default
 filesystem via 
\family typewriter
fsync()
\family default
, the application waiting for the write operation at 
\family typewriter
/dev/mars/mydata
\family default
 is signalled that the write was successful
\begin_inset Foot
status open

\begin_layout Plain Layout
In order to reclaim the temporary memory buffer, its content must be written
 back to the underlying disk 
\family typewriter
/dev/lv-x/mydat
\family default
a somewhen.
 After writeback, the temporary space is freed.
 The writeback can do the following optimizations:
\end_layout

\begin_layout Enumerate
writeback may be in 
\emph on
any
\emph default
 order; in particular, it is 
\emph on
sorted
\emph default
 according to ascending sector ´numbers.
 This reduces the average seek distances of magnetic disks in general.
\end_layout

\begin_layout Enumerate
when the same sector is overwritten multiple times, only the 
\begin_inset Quotes eld
\end_inset

last
\begin_inset Quotes erd
\end_inset

 version need to be written back, skipping some intermediate versions.
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
This usually happens 
\emph on
before
\emph default
 the writeback to the underlying LV 
\family typewriter
/dev/lv-x/mydata
\family default
 has started.
 Even when you power off the system right now, the information is not lost:
 it is present in the logfile, and will be reconstructed from there after
 restart from power loss (Recovery phase).
 This is similar to Recovery of database systems after unexpected power
 loss.
\end_layout

\begin_layout Standard
In case the primary node crashes during writeback, it suffices to replay
 the log entries from some point in the past until the end of the transaction
 logfile.
 It does no harm if you accidentally replay some log entries twice or even
 more often: since the replay is in the original total order.
 Thus any temporary inconsistency will be 
\emph on
healed
\emph default
 
\begin_inset Foot
status open

\begin_layout Plain Layout
\noindent
In mathematics, the property that you can apply your logfile twice to your
 data (or even as often as you want), is called 
\series bold
idempotence
\series default
.
 This is a very desirable property: it ensures that nothing goes wrong when
 replaying 
\begin_inset Quotes eld
\end_inset

too much
\begin_inset Quotes erd
\end_inset

 / starting your replay 
\begin_inset Quotes eld
\end_inset

too early
\begin_inset Quotes erd
\end_inset

.
 Idempotence is even more beneficial: in case anything should go wrong with
 your data on your disk (e.g.
 IO errors), replaying your logfile once more often may even 
\series bold
heal
\series default
 some defects.
\end_layout

\end_inset

by logfile application.
 Good news for desperate sysadmins forced to work with old or flaky hardware!
\end_layout

\begin_layout Standard
The basic idea of asynchronous replication by MARS is rather simple: just
 transfer the logfiles to your secondary nodes, and replay them onto their
 LV replica (aka copy of the disk data, aka mirror).
\end_layout

\begin_layout Standard
Replay is always in the same write order as the total order defined by the
 primary.
\end_layout

\begin_layout Standard
Therefore, a mirror of your data on any secondary may be outdated, but it
 always corresponds to some version which was valid in the past.
 This property is called 
\series bold
anytime consistency
\begin_inset Foot
status open

\begin_layout Plain Layout
Your secondary nodes are always 
\series bold
strictly consistent
\series default
 in themselves.
 Notice that this kind of consistency is a 
\emph on
local
\emph default
 consistency model.
 At global level, MARS is 
\series bold
eventually consistent
\series default
.
 Strict global consistency is generally not possible over long distances.
 Reasons are (1) Einstein's law (speed of light), and (2) the CAP theorem
 and its sister theorems.
 The front-cover pictures showing the planets Earth and Mars tries to lead
 your imagination away from global consistency models, in order to prepare
 you mentally for local consistency as in 
\begin_inset Quotes eld
\end_inset

MARS Think
\begin_inset Formula $^{(tm)}$
\end_inset


\begin_inset Quotes erd
\end_inset

.
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

As you can see in the picture, the logfile transfer process is 
\emph on
independent
\emph default
 from the replay process.
 Both processes can be switched on / off separately (see commands 
\family typewriter
marsadm {dis,}connect
\family default
 and 
\family typewriter
marsadm {pause,resume}-replay
\family default
 in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Operation-of-the"

\end_inset

).
 This may be 
\emph on
exploited
\emph default
: for example, you may replicate your logfiles as soon as possible (to protect
 against catastrophic failures), but deliberately wait one hour until it
 is replayed (under regular circumstances).
 If your data inside your filesystem 
\family typewriter
/mydata/
\family default
 at the primary site is accidentally destroyed by 
\family typewriter
rm -rf /mydata/
\family default
, you have an old copy at the secondary site.
 This way, you can substitute 
\emph on
some parts
\begin_inset Foot
status open

\begin_layout Plain Layout
Please note that MARS cannot 
\emph on
fully
\emph default
 substitute a backup system, because it can keep only 
\emph on
physical
\emph default
 copies, and does not create logical copies.
\end_layout

\end_inset


\emph default
 of conventional backup functionality by MARS.
 In case you need the actual version, just replay in 
\begin_inset Quotes eld
\end_inset

fast-forward
\begin_inset Quotes erd
\end_inset

 mode (similar to old-fashioned video tapes).
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Future versions of MARS are planned to also allow 
\begin_inset Quotes eld
\end_inset

fast-backward
\begin_inset Quotes erd
\end_inset

 rewinding, of course at some cost.
\end_layout

\begin_layout Section
The State of MARS
\begin_inset CommandInset label
LatexCommand label
name "sec:The-State-of"

\end_inset


\end_layout

\begin_layout Standard
In general, MARS tries to 
\emph on
hide
\emph default
 any network failures from you as best as it can.
 After a network problem, any internal low-level socket connections are
 
\emph on
transparently
\emph default
 tried to re-open ASAP, without need for sysadmin intervention.
 In difference to DRBD, network failures will 
\emph on
not
\emph default
 automatically alter the state of MARS, such as switching to 
\family typewriter
disconnected
\family default
 after a 
\family typewriter
ko_timeout
\family default
 or similar.
 From a high-level sysadmin viewpoint, communication may just take a very
 long time to succeed.
\end_layout

\begin_layout Standard
When the behaviour of MARS is different from DRBD, it is usually intended
 as a feature.
\end_layout

\begin_layout Standard
MARS is not only an 
\series bold
asynchronous
\series default
 system at block IO level, but also 
\series bold
at control level
\series default
.
\end_layout

\begin_layout Standard
This is 
\emph on
necessary
\emph default
 because in a widely distributed long-distance system running on slow or
 even temporarily failing networks, actions may take a long time, and there
 may be many actions 
\series bold
started in parallel
\series default
.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Synchronous concepts are generally not sufficient for expressing that.
 Because of inherent asynchronicity and of dynamic creation / joining of
 resources, it is neither possible to comprehensively depict a complex distribut
ed MARS system, nor a comprehensive standalone snippet of MARS, as a finite
 state transition diagram
\begin_inset Foot
status open

\begin_layout Plain Layout
Probably it could be possible to formally model MARS as a Petri net.
 However, complete Petri nets are tending to become very conplex, and to
 describe lots of low-level details.
 Expressing hierarchy, in a top-down fashion, is cumbersome.
 We find no clue in trying to do so.
\end_layout

\end_inset

.
\end_layout

\begin_layout Chapter
HOWTO setup MARS
\begin_inset CommandInset label
LatexCommand label
name "chap:Quick-Start-Guide"

\end_inset


\end_layout

\begin_layout Standard
This chapter is for impatient but experienced sysadmins.
 For more detailed information, refer to chapter 
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:The-Sysadmin-Interface"

\end_inset

.
\end_layout

\begin_layout Section
Description: what you Need
\begin_inset CommandInset label
LatexCommand label
name "sec:Preparation:-What-you"

\end_inset


\end_layout

\begin_layout Standard
This section describes the hardware you will need to buy and deploy, and
 which software components to install.
 Step-by-step setup instructions are following in the next section (starting
 with section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:MARS-Kernel-Module"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
\end_layout

\begin_layout Standard
Typically, you will install MARS at many bare metal servers for replication
 of many LVs 
\emph on
between
\begin_inset Foot
status open

\begin_layout Plain Layout
Many other solutions, even from commercial storage vendors, will not work
 reliably over distances greater than 
\begin_inset Formula $\approx50$
\end_inset

 km, and/or when your network is not 
\emph on
extremely
\emph default
 reliable, and/or when you try to push huge masses of data from high-performance
 applications through a network bottleneck.
 If you ever encountered suchalike problems (or try to avoid them in advance),
 MARS is for you.
 More information can be found in 
\family typewriter
mars-architecture-guide.pdf
\family default
.
\end_layout

\end_inset


\emph default
 multiple datacenters.
 Do 
\emph on
not
\emph default
 use MARS inside of VMs (see explanation of Dijkstra's layering rules in
 
\family typewriter
mars-architecture-guide.pdf
\family default
).
\end_layout

\begin_layout Standard
You can use MARS both at dedicated storage servers (e.g.
 for serving Windows clients over iSCSI), or at standalone Linux servers
 where CPU and storage are 
\emph on
not
\emph default
 separated.
\end_layout

\begin_layout Standard
Here is a list of software to be installed at your servers (with distro-specific
 tools like 
\family typewriter
dpkg
\family default
 / 
\family typewriter
aptitude
\family default
 / 
\family typewriter
rpm
\family default
 / 
\family typewriter
yum
\family default
 / 
\family typewriter
zypper
\family default
 / etc):
\end_layout

\begin_layout Itemize

\family typewriter
ssh
\end_layout

\begin_layout Itemize

\family typewriter
ssh-agent
\family default
 (such that 
\family typewriter
ssh root@hostA
\family default
 will work without password)
\end_layout

\begin_layout Itemize

\family typewriter
rsync
\end_layout

\begin_layout Itemize

\family typewriter
perl
\end_layout

\begin_layout Itemize

\family typewriter
lvm
\end_layout

\begin_layout Itemize
Further standard Linux tools like 
\family typewriter
modprobe
\family default
, typically already present at servers.
\end_layout

\begin_layout Itemize
Only if you don't have an already pre-built MARS kernel module, and only
 at your workstation, not necessarily at your server: everything you need
 for compiling a customized kernel.
 Optionally, the tools for building a Debian or rpm package.
 Details are distro-specific.
\end_layout

\begin_layout Standard
In order to protect your server data from low-level disk failures, you should
 use a 
\series bold
hardware RAID controller with BBU
\series default
.
 Software RAID is currently 
\emph on
not
\emph default
 recommended, because it generally provides worse performance due to the
 lack of a hardware BBU (for some benchmark comparisons with/out BBU, see
 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

https://github.com/schoebel/blkreplay/raw/master/doc/blkreplay.pdf
\end_layout

\end_inset

).
\end_layout

\begin_layout Standard
For many application workloads, RAID-6 provides a good compromize between
 cost and performance.
 Reads are very fast due to RAID-6 striping, while the slow RAID-6 writes
 are partially compensated by the MARS kernel memory buffer (see section
 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:IO-Performance-Tuning"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
\end_layout

\begin_layout Standard
For almost double the cost per TiB, you can speed up write operations by
 RAID-10.
 However, checkout RAID-6 first.
 A good tool to measure your 
\emph on
real
\emph default
 application performance is 
\family typewriter
blktrace
\family default
 plus blkreplay, see 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

https://github.com/schoebel/blkreplay/raw/master/doc/blkreplay.pdf
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
For much higher cost per TiB, typically by about a factor of 10, you can
 of course also use SSDs in place of HDDs.
 While relatively small-sized database workloads are nowadays typically
 on SSDs, big mass data is typically remaining on HDDs for cost reasons.
\end_layout

\begin_layout Standard
Typically, you should build more than one RAID set
\begin_inset Foot
status open

\begin_layout Plain Layout
For low-cost storage, RAID-5 is no longer regarded safe for today's typical
 storage sizes, because the error rate is regarded too high.
 Therefore, use RAID-6.
 If you need more than 15 disks in total, create multiple RAID sets (each
 having at most 15 disks, better about 12 disks) and stripe them via LVM
 (or via your hardware RAID controller if it supports RAID-60).
\end_layout

\end_inset

 if you have more than 12 to 15 spindles in total.
 Therefore, the step-by-instructions of this manual will show you some examples
 with LVM striping over 2 physical volumes (PVs).
\end_layout

\begin_layout Standard
LVM is highly recommended
\begin_inset Foot
status open

\begin_layout Plain Layout
In principle, you may combine MARS with commercial storage boxes connected
 over Fibrechannel or iSCSI.
 At 1&1, there is not yet operational experience with such setups.
\end_layout

\end_inset

 for maximum flexibility.
 When used in static space allocation mode (as opposed to thin provisioning
 mode), LVM involves no measurable overhead (within the measurement tolerances
 of 
\family typewriter
blkreplay
\family default
).
 Although LVM thin provisioning could potentially save some cost, it may
 lead to massive performance degradation as observed with certain types
 of application behaviour.
 In order to stay at the safe side of operations, you should dimension your
 RAID storage size accordingly.
\end_layout

\begin_layout Standard
MARS' tolerance of networking problems comes with some cost.
 You will need some extra space for the transaction logfiles of MARS, residing
 at the 
\family typewriter
/mars/
\family default
 filesystem.
\end_layout

\begin_layout Standard
The exact space requirements for 
\family typewriter
/mars/
\family default
 depend on the 
\emph on
average write rate
\emph default
 of your application, not on the size of your data.
 An example: in 1&1 Shared Hosting Linux (ShaHoLin), we found that only
 few applications are writing more than 1 TB per day during ordinary
\begin_inset Foot
status open

\begin_layout Plain Layout
Exception: restores from backup.
\end_layout

\end_inset

 operations.
 Most are writing even less than 100 GB per day, because the observed average
 filesystem data change rate is only about 1% per day
\begin_inset Foot
status open

\begin_layout Plain Layout
Within some limits, the distribution is an exponential one, according to
 Zipf's law.
\end_layout

\end_inset

.
 Of course, there exist other applications like backup where the write rate
 is much higher.
 Please try to determine your actual write rates from system tools like
 
\family typewriter
sar
\family default
.
 Usually, you want to dimension 
\family typewriter
/mars/
\family default
 such that you can survive a network loss lasting 3 days / about one weekend.
\end_layout

\begin_layout Standard
This can be achieved rather easily, in one of the following ways:
\end_layout

\begin_layout Enumerate
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
sloppy
\end_layout

\end_inset

 Create an LV for 
\family typewriter
/mars
\family default
 on top of your application VG, typically named 
\family typewriter
/dev/vg/mars
\family default
 or similar (see step-by-step instructions in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Setup-LVM"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
 This is the easiest solution if you are anyway using LVM on top of a hardware
 BBU.
 This is also most flexible: it can be 
\series bold
resized during operation
\series default
.
 Therefore, you may start with a size of around 500 GiB, and later be extended
 with increasing demands.
\begin_inset Newline newline
\end_inset

This variant is also recommended if you have very expensive SSD storage.
 Depending on write rates, you could for example start with 100 GiB, and
 extend dynamically as far as needed, for example by some alerting scripts,
 or even using some cron job.
\end_layout

\begin_layout Enumerate
Alternatively, you may use one 
\series bold
dedicated HDD
\series default
 with a capacity of 4 TB or more.
 Typically, this will provide you with plenty of headroom even for bigger
 networking incidents.
 Performace of a single HDD over a BBU is typically good enough for 
\family typewriter
/mars
\family default
 because the transaction logs are involving mostly 
\emph on
sequential
\emph default
 reads and writes in larger chunks.
 However, there exist some workloads where striping could be necessary for
 maximizing sequential throughput.
\end_layout

\begin_layout Enumerate
Alternatively, if you are concerned about both performance and reliability,
 use two dedicated spindles over hardware RAID-1 with BBU.
 For maximum flexibility, put another VG on top of the dediactedRAID-1 set.
 For example, if 
\family typewriter
/dev/sdc
\family default
 is your RAID-1 set, create a PV and a VG called 
\family typewriter
mars
\family default
 on top of it.
 This is most flexible, since you might later migrate your 
\family typewriter
/mars
\family default
 even during runtime, for example when replacing small disks with bigger
 ones, or when replacing HDDs with SSDs during runtime.
\end_layout

\begin_layout Enumerate
For extemely high performance, separate SSD sets for the user data VG and
 for 
\family typewriter
/mars
\family default
 might be beneficial.
 However, check whether it really pays off.
 Notice that a hardware BBU is nothing but a RAM cache, which is faster
 than any SSD, and there 
\emph on
exist
\emph default
 some workloads where sequntial IO to HDDs is faster than to SSDs.
 Sometimes, there are hidden performance bottlenecks, such as SAS busses,
 or some old-generation RAID controllers.
\end_layout

\begin_layout Standard
Dedicated HDDs for 
\family typewriter
/mars/
\family default
 have another advantage: their mechanical head movement is completely independen
t from your data head movements.
 For best performance, attach the corresponding disks to your hardware RAID
 controller with BBU, building a separate RAID set (even if it consists
 only of a single disk – notice that the 
\series bold
hardware BBU
\series default
 is the crucial point).
\end_layout

\begin_layout Standard
If you are concerned about reliability, use two disks configured as a relatively
 small RAID-1 set.
 For extremely high performance demands, you may consider (and check) RAID-10
 and/or SSD storage.
 However, SSDs are reported as less reliable.
 While failures of HDDs are typically detectable in advance by upcoming
 SMART media error counts, SSDs are typically failing suddenly and unexpectedly
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice: the component failure rate is not the crucial point.
 Even if some types of SSDs have a better MTBF than typical HDDs: when you
 can detect failure in advance, you can prevent
\end_layout

\end_inset

.
 And their failure is not statistically independent in general.
 Building a RAID-1 on top of SSDs bears an increased risk that 
\emph on
both
\emph default
 SSDs are unexpectedly failing both at the same time
\begin_inset Foot
status open

\begin_layout Plain Layout
Preliminary replacement of SSDs after a certain amount of write may help.
 But it will increase cost.
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
If you want to build extremely cheap low-cost storage, for example for low-perfo
rmance backup systems or similar use cases: cheap but high-capacity nearline-SAS
\begin_inset Foot
status open

\begin_layout Plain Layout
Even cheaper SATA disks are not recommended for professional datacenter
 usage.
 Typically, they are not rated for 24/7/365 usage.
 Even for some use cases like backup, experiences are worse.
\end_layout

\end_inset

 disks may be sufficient, because the transaction logfiles are highly sequential
 in their access pattern.
 However, check with 
\family typewriter
blkreplay
\family default
 that performance is 
\emph on
really
\emph default
 sufficient, when compared with 
\begin_inset Quotes eld
\end_inset

better
\begin_inset Quotes erd
\end_inset

 disks.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Do not import the block device for 
\family typewriter
/mars/
\family default
 over iSCSI.
 This would sacrifice both reliability and performance.
 MARS is constructed for exploiting a hardware BBU cache with a typical
 IO parallelism degree of 1000 parallel IO requests, over fast local DMA.
 See also section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:IO-Performance-Tuning"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Consequence: never run MARS inside of a VM (other than for functional component
 testing).
 See also Dijkstra's layering rules in 
\family typewriter
mars-architecture-guide.pdf
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Notice that the filesystem 
\family typewriter
/mars/
\family default
 has nothing to do with an ordinary filesystem.
 It is completely reserved for MARS internal purposes, namely as a 
\series bold
storage container
\series default
 for MARS' persistent data.
 It does not obey any userspace rules like FHS (filesystem hierarchy standard),
 and it should not be accessed by any userspace tool execpt the official
 
\family typewriter
marsadm
\family default
 tool.
 Its internal data format should be a regarded as a 
\series bold
blackbox
\series default
 by you.
 The internal data format may change in future, or the complete 
\family typewriter
/mars/
\family default
 filesystem may be even replaced by a totally different container format,
 while the official 
\family typewriter
marsadm
\family default
 interface and its primitive macros are supposed to remain stable.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

That said, you might look into its contents 
\emph on
by hand
\emph default
 for curiosity or for 
\emph on
debugging purposes
\emph default
, and only as root.
 But don't program any tools / monitoring scripts / etc bypassing the official
 
\family typewriter
marsadm
\family default
 tool.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Like DRBD, the current version of MARS has 
\series bold
no security
\series default
 built in.
 MARS assumes that it is running in a 
\series bold
trusted network
\series default
.
 Anyone who can connect to the MARS ports (default 7777 to 7779) can potentially
 breach in and become root.
 Therefore, you 
\series bold
must
\series default
 protect your network by appropriate means, such as firewalling and/or encrypted
 VPN.
\end_layout

\begin_layout Standard
Currently, MARS provides no shared secret like DRBD, because a simple shared
 secret is way too weak to provide any real security (potentially misleading
 people about the real level of security).
 Future versions of MARS might provide some 2-factor authorization, and
 encryption via dynamic session keys.
 Until that is implemented
\begin_inset Foot
status open

\begin_layout Plain Layout
There is fundamental argument: network traffic between datacenters belongs
 to a higher level than a single component like MARS.
 Thus its security requirements must be solved at that higher level, but
 not at the lower level of MARS.
\end_layout

\end_inset

, use a secured VPN instead.
 And don't forget to 
\emph on
audit
\emph default
 it for security holes.
\end_layout

\begin_layout Section
MARS Kernel Module
\begin_inset CommandInset label
LatexCommand label
name "sec:MARS-Kernel-Module"

\end_inset


\end_layout

\begin_layout Standard
Always use the newest stable version (master branch) from 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

https://github.com/schoebel/mars
\end_layout

\end_inset

.
 Please consult the file 
\family typewriter
ChangeLog
\family default
 there.
\end_layout

\begin_layout Standard
The MARS kernel module should be available or can be built via one of the
 following methods:
\end_layout

\begin_layout Itemize
As an external Debian or rpm kernel module, as provided by a package contributor
 (or hopefully by standard distros in the future).
\end_layout

\begin_layout Itemize
Via 
\family typewriter
dkms
\family default
.
 Although there is an example file 
\family typewriter
contrib/mars-dkms.dkms
\family default
 in the official MARS repo at 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

https://github.com/schoebel/mars
\end_layout

\end_inset

 which could serve as a base for Linux distro vendors, it would suffer from
 
\emph on
serious performance degradation
\emph default
 if it would be compiled 
\emph on
without
\emph default
 the MARS pre-patch.
 Don't use such a version for any serious application.
 With pre-patch, 
\family typewriter
dkms
\family default
 would be no problem.
 You can check whether some already built 
\family typewriter
mars.ko
\family default
 kernel module has been compiled with pre-patch or not:
\begin_inset Newline newline
\end_inset


\family typewriter
modinfo mars
\family default

\begin_inset Newline newline
\end_inset

which displays the version for the currently active kernel, or any other
 version via
\begin_inset Newline newline
\end_inset


\family typewriter
modinfo /path/to/mars.ko
\family default

\begin_inset Newline newline
\end_inset

This should display something like
\begin_inset Newline newline
\end_inset


\begin_inset listings
inline false
status open

\begin_layout Plain Layout

io_driver:      aio
\end_layout

\begin_layout Plain Layout

prepatch:       has_prepatch 
\end_layout

\end_inset

Do not use a kernel module for production if the io_driver either reports
 
\family typewriter
sio
\family default
 in place of 
\family typewriter
aio
\family default
, or if no pre-patch is detected.
\end_layout

\begin_layout Itemize
As a 
\emph on
separate
\emph default
 kernel module, only recommended for 
\emph on
experienced
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
You should be familiar with the problems arising from orthogonal combination
 of different kernel versions with different MARS module versions and with
 different 
\family typewriter
marsadm
\family default
 userspace tool versions at the package management level.
 Hint: 
\family typewriter
modinfo
\family default
 is your friend.
\end_layout

\end_inset

 sysadmins: see file 
\family typewriter
Makefile.dist
\family default
 (tested with some older versions of Debian; may need some extra work with
 other distros).
\end_layout

\begin_layout Itemize
Following are recommended build instructions for senior sysadmins or developers,
 inplace in the kernel source tree.
 Look into the subdirectory 
\family typewriter
pre-patches/
\family default
 of the MARS repo for the right version of the pre-patches.
\end_layout

\begin_layout Standard
Here are example instructions for LTS kernel 4.4, building everything from
 scratch at your Linux workstation, using 
\family typewriter
git
\family default
.
 Actions marked 
\begin_inset Quotes eld
\end_inset

for safety
\begin_inset Quotes erd
\end_inset

 are not generally necessary, but may be appropriate for recovery from previous
 build failures.
\end_layout

\begin_layout Enumerate

\family typewriter
git clone git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
 linux-stable.git
\end_layout

\begin_layout Enumerate

\family typewriter
cd linux-stable.git
\end_layout

\begin_layout Enumerate

\family typewriter
git checkout linux-4.4.y
\end_layout

\begin_layout Enumerate
For safety:
\begin_inset Newline newline
\end_inset


\family typewriter
git pull
\end_layout

\begin_layout Enumerate
Get an appropriate old 
\family typewriter
.config
\family default
 file.
 For example at some OpenSuSE distro:
\begin_inset Newline newline
\end_inset


\family typewriter
cp /boot/config-4.12.14-lp151.28.13-default .config
\family default

\begin_inset Newline newline
\end_inset

Notice: there are several other methods which are outside the scope of this
 manual.
 When in doubt, consult somebody with kernel build experience.
\end_layout

\begin_layout Enumerate

\family typewriter
make oldconfig || make olddefconfig
\end_layout

\begin_layout Enumerate

\family typewriter
cd block/
\end_layout

\begin_layout Enumerate

\family typewriter
git clone --recurse-submodules https://github.com/schoebel/mars
\end_layout

\begin_layout Enumerate
For safety, ensure you always have the newest MARS version:
\begin_inset Newline newline
\end_inset


\family typewriter
cd mars
\family default
/
\begin_inset Newline newline
\end_inset


\family typewriter
git checkout master
\begin_inset Newline newline
\end_inset

git pull
\begin_inset Newline newline
\end_inset

cd ..
\end_layout

\begin_layout Enumerate
Go back to the root of Linux git:
\begin_inset Newline newline
\end_inset


\family typewriter
cd ..
\end_layout

\begin_layout Enumerate
To avoid mixup of later actions with upstream patches:
\family typewriter

\begin_inset Newline newline
\end_inset

git branch -D mars-patches-for-4.4 || echo IGNORE THIS ERROR
\begin_inset Newline newline
\end_inset

git checkout -b mars-patches-for-4.4
\end_layout

\begin_layout Enumerate
Apply 
\emph on
all(!)
\emph default
 the patches from 
\family typewriter
block/mars/pre-patches/vanilla-$version
\family default
.
 For example with kernel 4.4:
\end_layout

\begin_deeper
\begin_layout Enumerate
For safety:
\family typewriter

\begin_inset Newline newline
\end_inset

git reset --hard; git clean -f
\end_layout

\begin_layout Enumerate

\family typewriter
git am block/mars/pre-patches/vanilla-4.4/*.patch
\end_layout

\end_deeper
\begin_layout Enumerate

\family typewriter
make menuconfig
\family default
 (or another variant like 
\family typewriter
make xconfig
\family default
)
\end_layout

\begin_deeper
\begin_layout Enumerate
Go to 
\begin_inset Quotes eld
\end_inset

Enable the block layer
\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Enumerate
Select MARS as a module (by moving the cursor and then typing 
\family typewriter
m
\family default
)
\end_layout

\begin_layout Enumerate
A lot of sub-options for MARS will pop up.
 Leave them at their default.
\end_layout

\begin_layout Enumerate
Save the Kconfig options to 
\family typewriter
.config
\family default
 and exit.
\end_layout

\end_deeper
\begin_layout Enumerate
Build the kernel, typically something like one of:
\end_layout

\begin_deeper
\begin_layout Enumerate
For Debian / Ubuntu / etc:
\begin_inset Newline newline
\end_inset


\family typewriter
make deb-pkg
\end_layout

\begin_layout Enumerate
For Redhat / CentOS / SuSE / etc:
\begin_inset Newline newline
\end_inset


\family typewriter
make rpm-pkg
\end_layout

\begin_layout Enumerate
Classical with local install:
\begin_inset Newline newline
\end_inset


\family typewriter
make -j 12 && make install modules_install
\end_layout

\begin_layout Enumerate
...
 see some more variants and make targets:
\begin_inset Newline newline
\end_inset


\family typewriter
make help
\end_layout

\end_deeper
\begin_layout Enumerate
Copy and install the new Debian or rpm package to the destination servers,
 activate them (system-dependent, like 
\family typewriter
update-grub
\family default
 or similar), and reboot your servers with the modified kernel.
\end_layout

\begin_layout Enumerate
Check that the MARS kernel module is installed (but do not yet load it):
\begin_inset Newline newline
\end_inset


\family typewriter
modinfo mars
\end_layout

\begin_layout Standard
Further / more accurate / latest instructions may be found in 
\family typewriter
README
\family default
 and in 
\family typewriter
INSTALL
\family default
.
 You must not only install the kernel and the 
\family typewriter
mars.ko
\family default
 kernel module to all of your bare metal cluster nodes, but also the 
\family typewriter
marsadm
\family default
 userspace tool.
\end_layout

\begin_layout Standard
Installing marsadm by hand is rather simple: just copy it to 
\family typewriter
/usr/bin/
\family default
 or 
\family typewriter
/usr/local/bin
\family default
.
 For example:
\end_layout

\begin_layout Itemize

\family typewriter
cp -a block/mars/userspace/marsadm /usr/local/bin/
\end_layout

\begin_layout Itemize
Alternatively, create a Debian or rpm package (named 
\family typewriter
mars-utils
\family default
 or 
\family typewriter
mars-tools
\family default
 or similar).
 Since 
\family typewriter
marsadm
\family default
 is a basic Perl script which requires no other Perl modules, the only package
 dependency is from the Perl interpreter.
 Since Perl is typically already installed on virtually every Linux server,
 leaving out this dependency won't be a show stopper.
\end_layout

\begin_layout Standard
With newer versions of MARS, a prepatch for vanilla kernels 3.2 through 4.9
 (or even later) is no longer needed, at least in theory.
 However, 
\series bold
IO performance
\series default
 is currently 
\emph on
much
\emph default
 worse when the pre-patch is not applied.
 This will be hopefully addressed in a future release.
 At the moment, don't use pre-patch-less MARS on any production system.
 It has extremely limited IO parallelism because some performance-critical
 kernel interfaces are not accessible without the pre-patch.
 The command 
\family typewriter
modinfo mars
\family default
 will report whether the pre-patch is present at your kernel, or not (see
 description of 
\family typewriter
dkms
\family default
 above).
\end_layout

\begin_layout Standard
Pre-patches for various kernel version can be found in the 
\family typewriter
pre-patches/
\family default
 subdirectory of the MARS source tree.
 Following are the types of pre-patches:
\end_layout

\begin_layout Itemize

\family typewriter
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
sloppy
\end_layout

\end_inset

 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

0001-mars-minimum-pre-patch-for-mars.patch
\end_layout

\end_inset


\family default
 or similar.
 Please prefer this one (when present for your kernel version) in front
 of any old / deprecated 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

0001-mars-generic-pre-patch-for-mars.patch
\end_layout

\end_inset


\family default
 or similar.
 The latter should not be used anymore, except for testing or as an emergency
 fallback.
\end_layout

\begin_layout Itemize

\family typewriter
\begin_inset Flex URL
status open

\begin_layout Plain Layout

0002-mars-SPECIAL-for-in-tree-build.patch
\end_layout

\end_inset


\family default
 or similar.
 This is 
\emph on
only
\emph default
 needed when building the MARS kernel module together with all other kernel
 modules in a single 
\family typewriter
make
\family default
 pass.
 For separate external module builds (e.g.
 with 
\family typewriter
dkms
\family default
, or with 
\family typewriter
rpmbuild
\family default
 etc), this patch 
\emph on
must not
\emph default
 be applied (but the pre-patch 
\emph on
strongly should
\emph default
 if somehow possible).
 When using this patch, please apply the aforementioned pre-patch also,
 because your kernel will be patched anyway.
\end_layout

\begin_layout Itemize
For certain kernels like 4.14, some additional fixes may be necessary.
 These are in the respective 
\family typewriter
vanilla-*/
\family default
 subdirectory, indicated by filename 
\family typewriter
*fix*
\family default
.
 Sometimes, these are 
\emph on
absolutely
\emph default
 needed.
 For example, an endless loop in unfixed upstream vanilla kernels >= 4.10
 may occur upon network timeouts.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Starting from version 
\family typewriter
mars0.1stable56
\family default
 or 
\family typewriter
mars0.1abeta8
\family default
, 
\series bold
submodules
\series default
 have been added to the github repo of MARS.
 If you have an old git checkout, please say 
\family typewriter
git pull --recurse-submodules=yes
\family default
 or similar.
 Otherwise you may be missing an important future part of the MARS release,
 without notice (depending on your local 
\family typewriter
git
\family default
 version and its local configuration).
\end_layout

\begin_layout Section
Setup Primary and Secondary Cluster Nodes
\begin_inset CommandInset label
LatexCommand label
name "sec:Setup-Primary-and"

\end_inset


\end_layout

\begin_layout Standard
If you already have some production data on your bare metal servers via
 LVM, you may skip some of the following subsections.
\end_layout

\begin_layout Standard
In case your data is already replicated with DRBD, you may migrate to MARS
 (or even back from MARS to DRBD) if you use 
\emph on
external
\begin_inset Foot
status open

\begin_layout Plain Layout

\emph on
Internal
\emph default
 DRBD metadata should also work as long as the filesystem inside your block
 device / disk already exists and is not re-created.
 The latter would destroy the DRBD metadata, but even that will not hurt
 you really: you can always switch back to DRBD using 
\emph on
external
\emph default
 metadata, as long as you have some small spare space somewhere.
\end_layout

\end_inset


\emph default
 DRBD metadata (which is not touched by MARS).
 Internal DRBD metadata is reported to also work, because it resides at
 the end of the block device.
 However, you will waste some small amount of storage.
\end_layout

\begin_layout Standard
Migrating back to DRBD is also possible, provided you re-initialize the
 DRBD meta-data again.
\end_layout

\begin_layout Standard
For the following instructions to work, you must be 
\family typewriter
root
\family default
 on your servers.
\end_layout

\begin_layout Subsection
Setup Hardware
\begin_inset CommandInset label
LatexCommand label
name "subsec:Setup-Hardware"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Do not use MARS inside of VMs.
 Only use at bare metal!
\end_layout

\begin_layout Standard
When using hardware RAID controllers with hardware BBU (as is highly recommended
), you will need to build your RAID sets with the corresponding tools.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Don't set your hardware BBU cache to 
\begin_inset Quotes eld
\end_inset

writethrough
\begin_inset Quotes erd
\end_inset

 mode.
 This may lead to tremendous performance degradation.
 Use the default 
\begin_inset Quotes eld
\end_inset

writeback
\begin_inset Quotes erd
\end_inset

 strategy instead.
 It should be operationally safe, because in case of power loss the BBU
 cache content will be preserved thanks to the battery, and/or thanks to
 goldcaps for saving the cache content into some flash chips.
\end_layout

\begin_layout Standard
In the following sections, we assume that two RAID sets are already built,
 and are accessible as 
\family typewriter
/dev/sdb
\family default
 and 
\family typewriter
/dev/sdc
\family default
.
\end_layout

\begin_layout Subsection
Setup LVM
\begin_inset CommandInset label
LatexCommand label
name "subsec:Setup-LVM"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Execute the following instructions only once after bare metal hardware
 deployment, or if you want to re-install your server.
 Otherwise, you may delete existing data.
\end_layout

\begin_layout Enumerate
First step is create the LVM meta-information on the RAID sets 
\family typewriter
/dev/sdc
\family default
 and 
\family typewriter
/dev/sdc
\family default
:
\begin_inset Newline newline
\end_inset


\family typewriter
pvcreate /dev/sdb
\family default

\begin_inset Newline newline
\end_inset


\family typewriter
pvcreate /dev/sdc
\end_layout

\begin_layout Enumerate
Check your physical volumes:
\begin_inset Newline newline
\end_inset


\family typewriter
pvs
\end_layout

\begin_layout Enumerate
Create a volume group:
\begin_inset Newline newline
\end_inset


\family typewriter
vgcreate vg /dev/sdb /dev/sdc
\end_layout

\begin_layout Enumerate
Check your volume group:
\begin_inset Newline newline
\end_inset


\family typewriter
vgs
\end_layout

\begin_layout Enumerate
Create a LV for /mars:
\begin_inset Newline newline
\end_inset


\family typewriter
lvcreate -i 2 -L 100G -n mars lv
\end_layout

\begin_layout Enumerate
Check your list of LVs:
\begin_inset Newline newline
\end_inset


\family typewriter
lvs
\end_layout

\begin_layout Subsection
Setup Cluster Nodes
\begin_inset CommandInset label
LatexCommand label
name "subsec:Setup-your-Cluster"

\end_inset


\end_layout

\begin_layout Standard
For your cluster, you need at least two bare metal nodes.
 In the following, they will be called hostA and hostB.
 In the beginning, hostA will have the 
\family typewriter
primary
\family default
 role, while hostB will be your initial 
\family typewriter
secondary
\family default
.
 The roles may change later.
\end_layout

\begin_layout Enumerate
On each of hostA and hostB, create the 
\family typewriter
/mars/
\family default
 mountpoint:
\begin_inset Newline newline
\end_inset


\family typewriter
mkdir /mars
\end_layout

\begin_layout Enumerate
On each host, create an 
\family typewriter
ext4
\family default
 filesystem on your separate disk / RAID set via 
\family typewriter
mkfs.ext4
\begin_inset Foot
status open

\begin_layout Plain Layout
Don't use 
\family typewriter
xfs
\family default
 for 
\family typewriter
/mars
\family default
.
 Its late allocation strategy may lead to deadlocks and other problems,
 at least with some elder kernel versions.
\end_layout

\end_inset


\family default
 .
\begin_inset Newline newline
\end_inset


\family typewriter
mkfs.ext4 /dev/vg/mars
\end_layout

\begin_layout Enumerate
On each host, mount that filesystem to 
\family typewriter
/mars/
\family default
.
 It is advisable to add an entry to 
\family typewriter
/etc/fstab
\family default
., or to create a systemd unit 
\family typewriter
mars.mount
\family default
.
 Here we just mount it by hand:
\begin_inset Newline newline
\end_inset


\family typewriter
mount /dev/vg/mars /mars
\end_layout

\begin_layout Enumerate
For security reasons, execute 
\family typewriter
chmod 0700 /mars
\family default
 everyhwere after 
\family typewriter
/mars/
\family default
 has been mounted.
 If you forget this step, any following 
\family typewriter
marsadm
\family default
 command will drop you a warning, but will fix the problem for you.
\end_layout

\begin_layout Enumerate
On hostA:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm create-cluster
\family default

\begin_inset Newline newline
\end_inset

This must be done 
\emph on
exactly once
\emph default
, on exactly one node of your cluster.
 Never do this twice or on different hosts, because that would create two
 different clusters which would have nothing to do with each other.
 The 
\family typewriter
marsadm
\family default
 tool protects you against accidentally joining / merging two different
 clusters.
 If you accidentally created two different clusters, just umount that 
\family typewriter
/mars/
\family default
 partition and start over with step 2.
\end_layout

\begin_layout Enumerate
Depending on the MARS version:
\end_layout

\begin_deeper
\begin_layout Enumerate
When using mars version mars0.1astable101 or later, execute on 
\emph on
both
\emph default
 hosts hostA and hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
modprobe mars
\end_layout

\begin_layout Enumerate
Old versions of MARS before mars0.1astable101 needed a working 
\family typewriter
ssh
\family default
 connection from hostB to hostA (as 
\family typewriter
root
\family default
).
 When needded, test ssh on hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
ssh hostA w
\begin_inset Newline newline
\end_inset


\family default
This should work without entering a password.
 Hint: you may use 
\family typewriter
ssh-agent
\family default
 and 
\family typewriter
ssh -A
\family default
 for achieving that.
\begin_inset Newline newline
\end_inset

In addition, 
\family typewriter
rsync
\family default
 must be installed.
\begin_inset Newline newline
\end_inset

Notice: in the old version, you 
\emph on
must not
\emph default
 modprobe before join-cluster is executed.
 In the new version, it is vice versa.
\end_layout

\end_deeper
\begin_layout Enumerate
On hostB
\family typewriter
:
\begin_inset Newline newline
\end_inset

marsadm join-cluster hostA
\end_layout

\begin_layout Enumerate
When not yet done, do on each of your hosts:
\begin_inset Newline newline
\end_inset


\family typewriter
modprobe mars
\end_layout

\begin_layout Enumerate
Check that mars is running everywhere:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm view all
\end_layout

\begin_layout Enumerate
Ignore any warnings that no resources are yet defined.
 But you should check that no warnings about network connections are appearing.
 Both cluster nodes should be able to communicate with each other over the
 MARS ports (default 7777 to 7779).
\end_layout

\begin_layout Enumerate
Additional checks, to be executed on 
\emph on
all
\emph default
 of your hosts:
\begin_inset Newline newline
\end_inset


\family typewriter
netstat -lp --tcp | grep 777
\begin_inset Newline newline
\end_inset

netstat --tcp | grep 777
\begin_inset Newline newline
\end_inset


\family default
Both variants should show up some healty connections.
 If not, fix your network configuration and/or firewalling etc.
 Details are outside of the scope of this manual.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Beware of asymmetric connections, caused by inappropriate firewall rules.
 
\emph on
Any
\emph default
 host must be able to communicate with 
\emph on
any
\emph default
 other host of the 
\emph on
same
\emph default
 cluster.
\end_layout

\begin_layout Section
Setup Housekeeping Cron Job
\begin_inset CommandInset label
LatexCommand label
name "subsec:Logfile-Rotation"

\end_inset


\end_layout

\begin_layout Standard
As explained in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:The-Transaction-Logger"

\end_inset

, all changes to your resource data are recorded in transaction logfiles
 residing on the 
\family typewriter
/mars/
\family default
 filesystem.
 These files are always growing over time.
 In order to avoid filesystem overflow, the following must be executed in
 regular time intervals:
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm cron
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Best practice is to run 
\family typewriter
marsadm cron
\family default
 in a 
\family typewriter
cron
\family default
 job, such as 
\family typewriter
/etc/cron.d/mars
\family default
.
 An example cronjob can be found in the 
\family typewriter
userspace/cron.d/
\family default
 subdirectory of the git repo.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

In addition, you should establish some regular monitoring of the free space
 present in the 
\family typewriter
/mars/
\family default
 filesystem.
\end_layout

\begin_layout Standard
More detailed information about about avoidance of 
\family typewriter
/mars/
\family default
 overflow is in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Defending-Overflow"

\end_inset

.
\end_layout

\begin_layout Standard
Here is some more background information if you want to configure your system
 cronjob manually.
 In most installations, a 10 minute cron interval should be sufficient.
 Here is an example line, to be placed in a file like 
\family typewriter
/etc/cron.d/mars
\family default
:
\end_layout

\begin_layout Enumerate

\family typewriter
*/10 * * * * root if [ -L /mars/uuid ] ; then marsadm cron ; fi > /dev/null
 2>&1
\end_layout

\begin_layout Standard
Here is some background explanation about some internal intermediate steps,
 as executed by 
\family typewriter
marsadm cron
\family default
.
 The following is not needed for operations, but might be helpful for testing
 and debugging.
 You can skip it if you don't have much time:
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm log-rotate all
\family default

\begin_inset Newline newline
\end_inset

This starts appending to a new logfile on all of your resources.
 The logfiles are automatically numbered by an increasing 9-digit logfile
 number.
 This will suffice for many centuries even if you would logrotate once a
 minute.
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm log-delete-all all
\family default

\begin_inset Newline newline
\end_inset

This determines all logfiles from all resources which are no longer needed
 (i.e.
 which are 
\emph on
fully
\emph default
 replayed, on 
\emph on
all
\emph default
 relevant secondaries).
 All superfluous logfiles are then deleted, including all copies on all
 secondaries.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 The current version of MARS deletes either 
\emph on
all
\emph default
 replicas of a logfile everywhere, or 
\emph on
none
\emph default
 of the replicas.
 This is a simple rule, but has the drawback that one node may hinder other
 nodes from freeing space in 
\family typewriter
/mars/
\family default
.
 In particular, the command 
\family typewriter
marsadm pause-replay $res
\family default
 (as well as 
\family typewriter
marsadm disconnect $res
\family default
) will freeze the space reclamation in the whole cluster when the pause
 is lasting very long.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 During such space accumulation, also the number of so-called deletions
 will accumulate in /mars/todo-global/ and sibling directories.
 In very big installations consisting of thousands of nodes, it is a good
 idea to regularly monitor the number of deletions similarly to the following:
 
\family typewriter
$(find /mars/ -name 
\begin_inset Quotes eld
\end_inset

delete-*
\begin_inset Quotes erd
\end_inset

 | wc -l)
\family default
 should not exceed a limit of ~150 entries.
\end_layout

\begin_layout Standard
Please prefer the short form 
\family typewriter
marsadm cron
\family default
 as an equivalent to scripting two separate commands 
\family typewriter
marsadm log-rotate all
\family default
 and 
\family typewriter
marsadm log-delete-all all
\family default
.
 The short form is not only easier to remember, but also future-proof in
 case some new MARS features should be added.
\end_layout

\begin_layout Section
Creating and Maintaining Resources
\begin_inset CommandInset label
LatexCommand label
name "sec:Creating-and-Maintaining"

\end_inset


\end_layout

\begin_layout Standard
For the sake of simplicity, the underlying LV as well as its later logical
 resource name as well as its later virtual device name will all be named
 uniformly by the same suffix 
\family typewriter
mydata
\family default
.
 In general, you might name each of them differently, but suchalike is not
 recommended, since it may easily lead to confusion in larger installations.
\end_layout

\begin_layout Standard
You may have some already pre-existing 
\family typewriter
/dev/lv/mydata
\family default
 at the initially primary hostA.
 Before using it for MARS, it must be unused for any other purpose (such
 as being mounted, or used by DRBD, etc).
 MARS will require 
\series bold
exclusive access
\series default
 to it.
\end_layout

\begin_layout Standard
If 
\family typewriter
/dev/lv/mydata
\family default
 already exists and contains some data (e.g.
 previously used by DRBD), you should skip the first three steps, otherwise
 you may destroy your data.
\end_layout

\begin_layout Enumerate
On both hostA and hostB, create a LV.
 In this example, its size is 50G:
\begin_inset Newline newline
\end_inset


\family typewriter
lvcreate -i 2 -L 50G -n mydata lv
\end_layout

\begin_layout Enumerate
On each node, check that the new LV is occurring in each of the following
 lists:
\begin_inset Newline newline
\end_inset


\family typewriter
lvs
\end_layout

\begin_layout Enumerate
Only on hostA, you may create a new filesystem:
\begin_inset Newline newline
\end_inset


\family typewriter
mkfs.xfs /dev/vg/mydata
\family default

\begin_inset Newline newline
\end_inset

Alternatively, here is a variant for creation of a zfs filesystem:
\begin_inset Newline newline
\end_inset


\family typewriter
zpool create /dev/vg/mydata
\family default

\begin_inset Newline newline
\end_inset


\family typewriter
zpool export /dev/vg/mydata
\end_layout

\begin_layout Enumerate
Only on hostA, say
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm create-resource mydata /dev/lv/mydata
\family default

\begin_inset Newline newline
\end_inset

As a result, a directory 
\family typewriter
/mars/resource-mydata/
\family default
 will be created on hostA, containing some symlinks.
 hostA will automatically start in the primary role for this resource.
 Therefore, a new pseudo-device 
\family typewriter
/dev/mars/mydata
\family default
 will appear after a few seconds.
\begin_inset Newline newline
\end_inset

Note that the initial content of 
\family typewriter
/dev/mars/mydata
\family default
 will be 
\emph on
exactly
\emph default
 the same as in your pre-existing LV 
\family typewriter
/dev/lv/mydata
\family default
.
\begin_inset Newline newline
\end_inset

If you like, you may now use 
\family typewriter
/dev/mars/mydata
\family default
 for mounting your already pre-existing data, or for creating a fresh filesystem
, or for exporting via iSCSI, and so on.
 You may do so even before any other cluster node has joined the resource
 (so-called 
\begin_inset Quotes eld
\end_inset

standalone mode
\begin_inset Quotes erd
\end_inset

).
 But you can also do so later after setup of (one ore many) secondaries.
\end_layout

\begin_layout Enumerate
On hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm wait-cluster
\family default

\begin_inset Newline newline
\end_inset

Check that the directory 
\family typewriter
/mars/resource-mydata/
\family default
 and its symlink content is also appearing on hostB.
 If not, check your network and/or firewall setup.
\end_layout

\begin_layout Enumerate
On hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm join-resource mydata /dev/lv/mydata
\family default

\begin_inset Newline newline
\end_inset

As a result, the initial full-sync from node A to node B should start automatica
lly.
\end_layout

\begin_layout Enumerate
On hostB, check that the sync is running after a few seconds:
\begin_inset Newline newline
\end_inset


\family typewriter
watch marsadm view all
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Of course, now any old content 
\family typewriter
/dev/lv/mydata
\family default
 at hostB (and 
\emph on
only
\emph default
 there!) is overwritten by the version from hostA.
 This is just what you wanted to do by setting up MARS replication.
 If you didn't check that your old contents at hostB didn't contain any
 valuable data (or if you accidentally provided a wrong LV device argument),
 it is too late now.
 Therefore, double-check that you are running 
\family typewriter
create-resource
\family default
 and 
\family typewriter
join-resource
\family default
 at the right sides of your cluster, and with the right block device names.
 Accidental confusion of the right sides, or accidental confusion of LV
 names may overwrite valuable data with wrong data, or even with uninitialized
 trash
\begin_inset Foot
status open

\begin_layout Plain Layout
Trying to mount unintitialized LV data is bad practice.
 It may even crash your kernel.
\end_layout

\end_inset

.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 In order to reduce suchalike risks, 
\family typewriter
marsadm
\family default
 does some basic checks.
 It checks that the disk device argument is really a block device, and that
 exclusive access to it is possible, as well as some further safety checks,
 e.g.
 whether the size is big enough.
 Notice that bigger replica device sizes are allowed at secondaries, although
 then you will waste some space.
 In such a case, 
\family typewriter
marsadm view all
\family default
 will display a warning.
 This behaviour is necessary as an intermediate step for online resizing
 via 
\family typewriter
marsadm resize
\family default
.
\end_layout

\begin_layout Standard
MARS cannot know the 
\emph on
purpose
\emph default
 of your generic block device.
 MARS (as well as DRBD) is completely ignorant of the 
\emph on
contents
\emph default
 of a generic block device; it does not interpret it in any way.
 Therefore, you may use MARS (as well as DRBD) for mirroring Windows filesystems
, or raw devices from databases, or virtual machines, or whatever.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Check that state 
\family typewriter
Orphan
\family default
 is left after a while on hostB.
 Notice that 
\family typewriter
join-resource
\family default
 is only 
\emph on
starting
\emph default
 a new replica, but does not wait for its completion.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 By default, MARS uses the so-called 
\begin_inset Quotes eld
\end_inset

fast fullsync
\begin_inset Quotes erd
\end_inset

 algorithm.
 It works similar to 
\family typewriter
rsync
\family default
, first reading the data on both sides and computing an md5 checksum for
 each block.
 Heavy-weight data is only transferred over the long-distance network upon
 checksum mismatch.
 This is extremely fast if your data is already (almost) identical on both
 sides.
 Conversely, if you know in advance that your initial data is completely
 different on both sides, you may choose to switch off the fast fullsync
 algorithm via 
\family typewriter
echo 0 > /proc/sys/mars/do_fast_fullsync
\family default
 in order to save the additional IO overhead and network latencies introduced
 by the separate checksum comparison steps.
\end_layout

\begin_layout Enumerate
Optionally, only for experienced sysadmins who 
\emph on
really
\emph default
 know what they are doing: if you will create a 
\emph on
new
\emph default
 filesystem on 
\family typewriter
/dev/mars/mydata
\family default
 
\emph on
after(!)
\emph default
 having created the MARS resource as well as 
\emph on
after
\emph default
 having already joined it on every replica, you may abandon the fast fullsync
 phase 
\emph on
before
\emph default
 creating the fresh filesystem, because the old content of 
\family typewriter
/dev/mars/mydata
\family default
 will then be just garbage not used by the freshly created filesystem
\begin_inset Foot
status open

\begin_layout Plain Layout
It is 
\emph on
vital
\emph default
 that the transaction logfile contents created by 
\family typewriter
mkfs
\family default
 is 
\emph on
fully
\emph default
 propagated to the secondaries and then replayed there.
\end_layout

\begin_layout Plain Layout
Analogously, another exception is also possible, but at your own risk (be
 careful, really!): when migrating your data from DRBD to MARS, and you
 have ensured that (1) at the end of using DRBD both your replicas were
 really equal (you should have checked that), and (2) before and after setting
 up any side of MARS (
\family typewriter
create-resource
\family default
 as well as 
\family typewriter
join-resource
\family default
) nothing has been written at all to it (i.e.
 no usage, neither of 
\family typewriter
/dev/lv/mydata
\family default
 nor of 
\family typewriter
/dev/mars/mydata
\family default
 has occurred in any way), the first transaction logfile 
\family typewriter
/mars/resource-mydata/log-000000001-$primary
\family default
 created by MARS will be empty.
 Check whether this is really true! Then, and only then, you may also issue
 a 
\family typewriter
fake-sync
\family default
.
\end_layout

\end_inset

.
 Then, and only then, you may say 
\family typewriter
marsadm fake-sync mydata
\family default
 in order to abort the sync operation.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Never do a 
\family typewriter
fake-sync
\family default
 unless you are 
\series bold
absolutely sure
\series default
 that you really don't need to sync the data! Otherwise, you are 
\emph on
guaranteed
\emph default
 to have produced harmful inconsistencies.
 If you accidentally issued 
\family typewriter
fake-sync
\family default
, you may startover the fast full sync at your secondary side by saying
 
\family typewriter
marsadm invalidate mydata
\family default
 (analogously to the corresponding DRBD command).
\end_layout

\begin_layout Chapter
HOWTO operation of MARS resources
\begin_inset CommandInset label
LatexCommand label
name "chap:HOWTO-operation-of"

\end_inset


\end_layout

\begin_layout Section
Inspecting the State of MARS
\begin_inset CommandInset label
LatexCommand label
name "sec:Inspecting-the-State"

\end_inset


\end_layout

\begin_layout Standard
The main command for viewing the current state of MARS is
\end_layout

\begin_layout Itemize

\family typewriter
marsadm view mydata
\end_layout

\begin_layout Standard
or its more specialized variant
\end_layout

\begin_layout Itemize

\family typewriter
marsadm view-
\emph on
$macroname
\emph default
 mydata
\end_layout

\begin_layout Standard
where 
\family typewriter
\emph on
$macroname
\family default
\emph default
 is one of the macros described in the following section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "subsec:Standard-marsadm-view"
plural "false"
caps "false"
noprefix "false"

\end_inset

, or in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-macro-processor"
plural "false"
caps "false"
noprefix "false"

\end_inset

, or another macro which has been written by yourself.
\end_layout

\begin_layout Standard
You may replace the resource name 
\family typewriter
mydata
\family default
 with the special keyword 
\family typewriter
all
\family default
 in order to get the state of all locally joined resources, as well as a
 list of all those resources.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

When using the variant 
\family typewriter
marsadm view all
\family default
, additionally the global communication status will be displayed.
 This helps humans in diagnosing problems.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Hint: use the compound command 
\family typewriter
watch marsadm view all
\family default
 for continuous display of the current state of MARS.
 When starting this side-by-side in 
\family typewriter
ssh
\family default
 terminal windows for all your cluster nodes, you can easily watch what's
 going on in the whole cluster.
\end_layout

\begin_layout Subsection
Standard 
\family typewriter
marsadm view
\begin_inset CommandInset label
LatexCommand label
name "subsec:Standard-marsadm-view"

\end_inset


\end_layout

\begin_layout Standard
The following predefined complex macros try to address the information needs
 of 
\series bold
humans
\series default
.
 Use them only in scripts when you are prepared about the fact that the
 output format may change during development of MARS.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
default
\family default
 This is equivalent to 
\family typewriter
marsadm view mydata
\family default
 without 
\family typewriter
\emph on
-maroname
\family default
\emph default
 suffix.
 It shows a one-line status summary for each resource, optionally followed
 by informational lines such as progress bars whenever a sync or a fetch
 of logfiles is currently running.
 The status line has the following fields:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%{res}
\family default
 resource name.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
[
\emph on
this_count
\emph default
/
\emph on
total_count
\emph default
]
\family default
 total number of replicas of this resource, out of total number of cluster
 members.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%include{diskstate}
\family default
 see 
\family typewriter
diskstate
\family default
 macro below.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%include{replstate}
\family default
 see 
\family typewriter
replstate
\family default
 macro below.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%include{flags}
\family default
 see 
\family typewriter
flags
\family default
 macro below.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%include{role}
\family default
 see 
\family typewriter
role
\family default
 macro below.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%include{primarynode}
\family default
 see 
\family typewriter
primarynode
\family default
 macro below.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
%include{commstate}
\family default
 see 
\family typewriter
commstate
\family default
 macro below.
\end_layout

\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset space ~
\end_inset

 After that, optional lines such as progress bars are appearing only when
 something unusual is happening.
 These lines are subject to future changes.
 For examples, wasted disk space due to missing 
\family typewriter
resize
\family default
 is reported when 
\family typewriter
%{threshold}
\family default
 is exceeded.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Customization via your own macros (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Creating-your-own"

\end_inset

) is explicitly encouraged for experienced sysadmins and userspace developers.
 It would be nice if a vibrant user community would emerge, helping each
 other by exchange of macros.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Hint: in order to produce your own customized inspection / monitoring tools,
 you may ask the author for an official reservation of a macro sub-namespace
 such as 
\family typewriter
*-
\emph on
yourcompanyname
\family default
\emph default
.
 You will be fully responsible for your own reserved namespace and can do
 with it whatever you want.
 The official MARS release will guarantee that 
\emph on
no name clashes
\emph default
 with your reserved sub-namespace will occur in future.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
default-global
\family default
 Currently, this just calls 
\family typewriter
comminfo
\family default
 (see below).
 May be extended in future.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-info
\family default
 When present, shows the status of /dev/mars/mydata in human-readyble form.
 Shows the empty string when /dev/mars/mydata is not present.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
diskstate
\family default
 Shows the status of the underlying disk device, in the following order
 of precedence
\begin_inset Foot
status open

\begin_layout Plain Layout
When an earlier list item is displayed, no combinations with following items
 are possible.
 This kind of 
\begin_inset Quotes eld
\end_inset

hiding effect
\begin_inset Quotes erd
\end_inset

 can lead to an 
\emph on
information loss
\emph default
.
 In order to get a non-lossy picture from the state of your system, please
 look at the 
\family typewriter
flags
\family default
 which are able to display cartesian combinations of more detailed internal
 states.
\end_layout

\end_inset

:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NotJoined
\family default
 (cf 
\family typewriter
%get-disk{}
\family default
) No underlying disk device is configured.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NotPresent
\family default
 (cf 
\family typewriter
%disk-present{}
\family default
) The underlying disk device (as configured, see 
\family typewriter
marsadm view-get-disk
\family default
) does not exist or the device node is not accessible.
 Therefore MARS cannot work.
 Check that LVM or other software is properly configured and running.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Detached
\family default
 (cf 
\family typewriter
InConsistent
\family default
, 
\family typewriter
NeedsReplay
\family default
, 
\family typewriter
%todo-attach{}
\family default
, 
\family typewriter
%is-attach{}
\family default
) The underlying disk is willingly switched off (see 
\family typewriter
marsadm detach
\family default
), and it actually is no longer opened by MARS.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Detaching
\family default
 (cf 
\family typewriter
%todo-attach{}
\family default
 and 
\family typewriter
%is-attach{}
\family default
) Access to the underlying disk is switched off, but actually not yet 
\family typewriter
close()
\family default
d by MARS.
 This can happen for a long time on a primary when other secondaries are
 accessing the disk remotely for syncing.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
DefectiveLog[
\emph on
description-text
\emph default
]
\family default
 (cf 
\family typewriter
%replay-code{}
\family default
) Typicially this indicates an 
\family typewriter
md5
\family default
 checksum error in a transaction logfile, or another (hardware / filesystem)
 defect.
 This occurs extremely rarely in practice, but has been observed more frequently
 during a massive failure of air conditioning in a datacenter, when disk
 temperatures raised to more than 80° Celsius.
 Notice that a secondary 
\series bold
refuses
\series default
 to apply any knowingly defective logfile data to the disk.
 Although this message is 
\emph on
not directly
\emph default
 referring to the underlying disk, it is mentioned here because of its superior
 
\series bold
relevance
\series default
 for the diskstate.
 A damaged transaction logfile will always affect the 
\emph on
actuality
\emph default
 of the disk, but not its 
\emph on
integrity
\emph default
 (by itself).
 What to do in such a case?
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Enumerate
When the damage is only at one of your secondaries, you should first ensure
 that the primary has a good logfile after a 
\family typewriter
marsadm log-rotate
\family default
, then try 
\family typewriter
marsadm invalidate
\family default
 at the damaged secondary.
 It is crucial that the primary has a fresh correct logfile behind the error
 position, and that it is continuing to operate correctly.
\end_layout

\begin_layout Enumerate
When 
\emph on
all
\emph default
 of your secondaries are reporting 
\family typewriter
DefectiveLog
\family default
, the primary could have 
\emph on
produced
\emph default
 a damaged logfile (e.g.
 in RAM, in a DMA channel, etc) while continuing to operate, and all of
 your secondaries got that defective logfile.
 After 
\family typewriter
marsadm log-delete-all all
\family default
, you can check this by comparing the 
\family typewriter
md5sum
\family default
 of the first primary logfile (having the lowest serial number) with the
 versions on your replicas.
 The problem is that you don't know whether the primary side has a silent
 corruption on any of its disks, or not.
 You will need to take an operational decision whether to switchover to
 a secondary via 
\family typewriter
primary --force
\family default
, or whether to continue operation at the primary and 
\family typewriter
invalidate
\family default
 your secondaries.
\end_layout

\begin_layout Enumerate
When the original primary is affected in a very bad way, such that it crashed
 badly and afterwards even recovery of the 
\emph on
primary
\emph default
 is impossible
\begin_inset Foot
status open

\begin_layout Plain Layout
In such a rare case, the 
\emph on
original primary
\emph default
 (but not any other host) 
\series bold
refuses
\series default
 to come up during recovery with 
\emph on
his own
\emph default
 logfile originally produced by 
\emph on
himself
\emph default
.
 This is not a bug, but saves you from incorrectly assuming that your original
 primary disk were consistent - it is 
\emph on
known
\emph default
 to be inconsistent, but recovery is impossible due to the damaged logfile.
 Thus 
\emph on
this one
\emph default
 replica is trapped by defective hardware.
 The other replicas shouldn't.
\end_layout

\end_inset

 due to this error (which typically occurs extremely rarely, observed two
 times during 7 millions of operating hours on defective hardware), you
 need to take an operational decision between the following alternatives:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Enumerate
switchover to a former secondary via 
\family typewriter
primary --force
\family default
, producing a split brain, and producing some (typically small) data loss.
 However, integrity is more important than actuality in such an extreme
 case.
\end_layout

\begin_layout Enumerate
deconstruction of the resource at 
\emph on
all
\emph default
 replicas via 
\family typewriter
leave-resource --force
\family default
, running 
\family typewriter
fsck
\family default
 or similar tools by hand at the underlying disks, selecting the best replica
 out of them, and finally re-constructing the resource again.
\end_layout

\begin_layout Enumerate
restore your backup.
\end_layout

\end_deeper
\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Orphan
\family default
 The secondary cannot replay data anymore, because it has been kicked out
 for avoidance of emergency mode.
 The data is not recent anymore.
 Typically, 
\family typewriter
marsadm invalidate
\family default
 needs to be done.
\begin_inset Newline newline
\end_inset

There is an execption: shortly after 
\family typewriter
join-resource
\family default
 or 
\family typewriter
invalidate
\family default
, it may take some time until state 
\family typewriter
Orphan
\family default
 may be left, and until the newest logfile has appeared at your secondary
 site (depending on the size of logfiles, and on your network).
 In case of network problems, this may take very long.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 This state tells you that your replica is not current, and currently not
 being updated at all.
 Don't forget to 
\series bold
monitor
\series default
 for longer occurrences of this state! Otherwise you may get a big surprise
 when you need a forceful emergency failover, but your replica is very old
 or even does not really exist at all.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NoAttach
\family default
 (cf 
\family typewriter
%is-attach{}
\family default
) The underlying disk is currently not opened by MARS.
 Reasons may be that the kernel module is not loaded, or an exclusive 
\family typewriter
open()
\family default
 is currently not possible because somebody else has already opened it.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
InConsistent
\family default
 (cf 
\family typewriter
%is-consistent{}
\family default
) A logfile replay and/or sync is known to be needed / or to complete (e.g.
 after 
\family typewriter
invalidate
\family default
 has started) in order to restore local consistency (for details, look at
 
\family typewriter
flags
\family default
).
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Hint: in the current implementation of MARS, this will never happen on secondari
es during ordinary replay (but only when either sync has not yet finished,
 or when the 
\emph on
initial
\emph default
 logfile replay after the sync has not yet finished), because the ordinary
 logfile replay always maintains anytime consistency once a consistent state
 had been reached.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 
\emph on
Only
\emph default
 in case of a primary node crash, and 
\emph on
only
\emph default
 after attempts have failed to become primary again (e.g.
 IO errors, etc), this 
\emph on
can
\emph default
 (but need not) mean that something went wrong.
 Even in such an extremely unlikely event, chances are high that 
\family typewriter
fsck 
\family default
can fix any remaining problems (and, of course, you can also switchover
 to a former secondary).
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

When this message appears, simply start MARS again (e.g.
 
\family typewriter
modprobe mars; marsadm up all
\family default
), in whatever role you are intending.
 This will 
\emph on
automatically
\emph default
 go into phase 
\family typewriter
Recovery
\family default
, i.e.
 try to replay any necessary transaction logfile(s) in order to fix any
 inconsistency.
 Only if the automatic fix fails and this message persists for a long time
 without progress, you 
\emph on
might
\emph default
 have a problem.
 Typically, as observed at a large installation at 1&1, this happens extremely
 rarely, and even then it just indicated that hardware was defective.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
OutDated[FR]
\family default
 (cf 
\family typewriter
%work-reached{}
\family default
) Only at secondaries.
 Tells whether it is 
\emph on
currently known
\emph default
 that the disk has any lag-behind when compared to the 
\emph on
currently known
\emph default
 state of the current designated primary (if there exists one).
 Only meaningful if a current designated primary exists.
 Notice that this kind of status display is subject to 
\emph on
natural races
\emph default
, for example when new logfile data has been produced in parallel, or network
 propagation is very slow.
 Additional information is in brackets:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
[F]
\family default
 Fetch is known to be needed.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
[R]
\family default
 Replay is known to be needed.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
[FR]
\family default
 Both are known to be needed.
\end_layout

\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
WriteBack[
\emph on
amount
\emph default
]
\family default
 (cf 
\family typewriter
%is-primary{}
\family default
 and amount via 
\family typewriter
%writeback-rest{}
\family default
) Appears only at actual primaries (whether designated or not), when the
 writeback from the RAM buffer is active (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-Transaction-Logger"

\end_inset

).
 The 
\emph on
amount
\emph default
 is displayed in human readable form, and may be used for a very rough estimatio
n of recovery time after a primary crash.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Recovery
\family default
 (cf 
\family typewriter
%todo-primary{}
\family default
) Appears only at the designated primary before it actually has become primary.
 Similar to database recovery, this indicates the recovery phase after a
 crash
\begin_inset Foot
status open

\begin_layout Plain Layout
In some cases, 
\family typewriter
primary --force
\family default
 may also trigger this message.
\end_layout

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
EmergencyMode
\family default
 (cf 
\family typewriter
%is-emergency{}
\family default
) A current designated primary exists, and it is known that this host has
 entered emergency mode.
 See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Emergency-Mode"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
UpToDate
\family default
 Displayed when none of the above has been detected.
\end_layout

\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
replstate
\family default
 Shows the status of the replication in the following order of precedence:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
ModuleNotLoaded
\family default
 (cf 
\family typewriter
%is-module-loaded{}
\family default
) No kernel module is loaded, and as a consequence no 
\family typewriter
/proc/sys/mars/
\family default
 does exist.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
UnResponsive
\family default
 (cf 
\family typewriter
%is-alive{%{host}}
\family default
) The main thread 
\family typewriter
mars_main
\family default
 did not do any noticable work for more than 
\family typewriter
%{window}
\family default
 (default 60) seconds.
 Notice that this may happen when deleting 
\emph on
extremely
\emph default
 large logfiles (up to hundreds of gigabytes or terabytes).
 If this happens for a 
\emph on
very
\emph default
 long time, you should check 
\family typewriter
dmesg
\family default
 whether you might need a reboot in order to fix the hang.
 The time window may be changed by 
\family typewriter
--window=$seconds
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NotJoined
\family default
 (cf 
\family typewriter
%get-disk{}
\family default
) No underlying disk device is configured for this resource.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NotStarted
\family default
 (cf 
\family typewriter
%todo-attach{}
\family default
) Replication has not been started.
\end_layout

\begin_layout Itemize
When the current host is designated as a primary, the rest of the precedence
 list looks as follows:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
EmergencyMode
\family default
 (cf.
 
\family typewriter
%is-emergency{}
\family default
) See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Emergency-Mode"

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Replicating
\family default
 (cf.
 
\family typewriter
%is-primary{}
\family default
) Primary mode has been entered.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NotYetPrimary 
\family default
(catchall) This means the current host 
\emph on
should
\emph default
 act as a primary (see 
\family typewriter
marsadm primary
\family default
 or 
\family typewriter
marsadm primary --force
\family default
), but currently doesn't (yet).
 This happens during logfile replay, before primary mode is actually entered.
 Notice that replay of very big logfiles may take a long time.
\end_layout

\end_deeper
\begin_layout Itemize
When the current host is 
\emph on
not
\emph default
 designated as a primary:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
PausedSync
\family default
 (cf.
 
\family typewriter
%sync-rest{}
\family default
 and 
\family typewriter
%todo-sync{}
\family default
) Some data needs to be synced, but sync is currently switched off.
 See 
\family typewriter
marsadm {pause,resume}-sync
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Syncing
\family default
 (cf.
 
\family typewriter
%is-sync{}
\family default
) Sync is currently running.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
PausedFetch
\family default
 (cf.
 
\family typewriter
%todo{fetch}
\family default
) Fetch is currently switched off.
 See 
\family typewriter
marsadm {pause,resume}-fetch
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
PausedReplay
\family default
 (cf.
 
\family typewriter
%todo{replay}
\family default
) Replay is currently switched off.
 See 
\family typewriter
marsadm {pause,resume}-replay
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
NoPrimaryDesignated
\family default
 (cf.
 
\family typewriter
%get-primary{}
\family default
) A 
\family typewriter
secondary
\family default
 command has been given somewhere in the cluster.
 Thus no designated primary exists.
 All resource members are in state 
\family typewriter
Secondary
\family default
 or try to approach it.
 Sync and other operations are not possible.
 This state is therefore not recommended.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
PrimaryUnreachable
\family default
 (cf.
 
\family typewriter
%is-alive{}
\family default
) A current designated primary has been set, but this host has not been
 remotely updated for more than 60 seconds (see also 
\family typewriter
--window=$seconds
\family default
).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Orphan
\family default
 The secondary cannot replay data anymore, because it has been kicked out
 for avoidance of emergency mode.
 The data is not recent anymore.
 Typically, 
\family typewriter
marsadm invalidate
\family default
 needs to be done.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
Replaying
\family default
 (catchall) None of the previous conditions have triggered.
\end_layout

\end_deeper
\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
flags
\family default
 For each of disk, consistency, attach, sync, fetch, and replay, show exactly
 one character.
 Each character is either a capital one, or the corresponding lowercase
 one, or a dash.
 The meaning is as follows:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000
disk/device: 
\family typewriter
D
\family default
 = the device 
\family typewriter
/dev/mars/mydata
\family default
 is present, 
\family typewriter
d
\family default
 = only the underlying disk 
\family typewriter
/dev/lv-x/mydata
\family default
 is present, 
\family typewriter
-
\family default
 = none present / configured.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
consistency: this relates to the 
\emph on
underlying disk
\emph default
, not to 
\family typewriter
/dev/mars/mydata
\family default
! 
\family typewriter
C
\family default
 = locally consistent, 
\family typewriter
c
\family default
 = maybe inconsistent (no guarantee), - = cannot determine.
 Notice: this does not tell anything about 
\emph on
actuality
\emph default
.
 Notice: like the other flags, this flag is subject to races and therefore
 should be relied on only in 
\emph on
detached
\emph default
 state! See also description of macro 
\family typewriter
is-consistent
\family default
 below.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
attach: 
\family typewriter
A
\family default
 = attached, 
\family typewriter
a
\family default
 = currently trying to attach/detach but not yet ready (intermediate state),
 
\family typewriter
-
\family default
 = attach is switched off.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
sync: 
\family typewriter
S
\family default
 = sync finished, 
\family typewriter
s
\family default
 = currently syncing, 
\family typewriter
-
\family default
 = sync is switched off.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
fetch: 
\family typewriter
F
\family default
 = according to knowlege, fetched logfiles are up-to-date, 
\family typewriter
f
\family default
 = currently fetching (some parts of) a logfile, 
\family typewriter
-
\family default
 = fetch is switched off.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
replay: 
\family typewriter
R
\family default
 = all fetched logfiles are replayed, 
\family typewriter
r
\family default
 = currently replaying, 
\family typewriter
-
\family default
 = replay is switched off.
\end_layout

\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
todo-role
\family default
 Shows the 
\emph on
designated
\emph default
 state: 
\family typewriter
None
\family default
, 
\family typewriter
Primary
\family default
 or 
\family typewriter
Secondary
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
role
\family default
 Shows the 
\emph on
actual
\emph default
 state: 
\family typewriter
None
\family default
, 
\family typewriter
NotYetPrimary
\family default
, 
\family typewriter
Primary
\family default
, 
\family typewriter
RemainsPrimary
\family default
, 
\family typewriter
ForcedPrimary
\family default
 or 
\family typewriter
Secondary
\family default
.
 Any differences to the designated state are indicated by a prefix to the
 keyword 
\family typewriter
Primary
\family default
: 
\family typewriter
NotYet
\family default
 means that it 
\emph on
should
\emph default
 become primary, but actually hasn't.
 Vice versa, 
\family typewriter
Remains
\family default
 means that it 
\emph on
should
\emph default
 leave primary state in order to become secondary, but actually cannot do
 that because the 
\family typewriter
/dev/mars/mydata
\family default
 device is currently in use .
 
\family typewriter
ForcedPrimary
\family default
 indicates that 
\emph on
multiple
\emph default
 cluster hosts (see 
\family typewriter
%nr-primary{}
\family default
) are claiming to be in actual primary role, e.g.
 when another one is in role 
\family typewriter
RemainsPrimary
\family default
, or when network interruption is preventing role change information from
 propagating.
\begin_inset Newline newline
\end_inset


\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
%todo-primary{} == 0
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
%todo-primary{} == 1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
%is-primary{} == 0
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
None
\family default
 / 
\family typewriter
Secondary
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
NotYetPrimary
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
%is-primary{} == 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
RemainsPrimary
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
Primary / ForcedPrimary
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
primarynode
\family default
 Display 
\family typewriter
(none)
\family default
 or the hostname of the designated primary.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
commstate
\family default
 When the last metadata communication to the designated primary is longer
 ago than 
\family typewriter
${window}
\family default
 (see also 
\family typewriter
--window=
\emph on
seconds
\family default
\emph default
 option), display that age in human readable form.
 See also primitive macro 
\family typewriter
%alive-age{}
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
syncinfo
\family default
 Shows an informational progress bar when sync is running.
 Intended for humans.
 Scripts should not rely on any details from this.
 Scripts may use this only as an 
\emph on
approximate
\emph default
 means for detecting progress (when comparing the 
\emph on
full
\emph default
 output text to a prior version and finding 
\emph on
any
\emph default
 difference, they may conclude that some progress has happened, how small
 whatsoever).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
replinfo
\family default
 Shows an informational progress bar when fetch is running.
 This should not be used for scripting at all, because it contains realtime
 information in human-readable form.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
fetch-line
\family default
 Additional details, called by 
\family typewriter
replinfo
\family default
.
 Shows the amount of data to be fetched, as well as the current transfer
 rate and a very rough estimation of the future duration.
 When primitive macros 
\family typewriter
%fetch-age{}
\family default
 or 
\family typewriter
%fetch-lag{}
\family default
 exceed 
\family typewriter
${window}
\family default
, their values are also displayed for human informational purposes.
 See description of these primitive macros.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
replay-line
\family default
 Additional details, called by 
\family typewriter
replinfo
\family default
.
 Shows the amount of data to be replayed, as well as the current replay
 rate and a very rough estimation of the future duration.
 When primitive macro 
\family typewriter
%replay-age{}
\family default
 exceeds 
\family typewriter
${window}
\family default
, it is also displayed for human informational purposes.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
comminfo
\family default
 When the network communication is in an unusual condition, display it.
 Otherwise, don't produce any output.
\end_layout

\begin_layout Section
Switch Primary / Secondary Roles
\begin_inset CommandInset label
LatexCommand label
name "sec:Switch-Primary-/"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/switching.fig
	width 90col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
MARS distinguishes between 
\emph on
intended
\emph default
 and 
\emph on
forced
\emph default
 switching.
 This distinction is necessary due to the communication architecture (asynchrono
us communication vs synchronous communication, see explanation of Lamport
 Clock in 
\family typewriter
mars-for-kernel-developers.pdf
\family default
).
\end_layout

\begin_layout Standard
Asynchronous communication means that (in worst case) a message may take
 (almost) arbitrary time in a distorted network to propagate to another
 node.
 As a consequence, the risk for accidentally creating an (unintended) split
 brain is increased (as compared to a synchronous system like DRBD).
\end_layout

\begin_layout Standard
In order to minimize this risk, MARS has invested a lot of effort into an
 internal handover protocol when you start an 
\emph on
intended
\emph default
 primary switch.
\end_layout

\begin_layout Subsection
Intended Switching / Planned Handover
\begin_inset CommandInset label
LatexCommand label
name "subsec:Intended-Switching"

\end_inset


\end_layout

\begin_layout Standard
Before starting a planned handover from your old primary 
\family typewriter
hostA
\family default
 to a new primary 
\family typewriter
hostB
\family default
, you should check the replication of the resource.
 As a human, use 
\family typewriter
marsadm view mydata
\family default
.
 For scripting, use the macros from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Predefined-Trivial-Macros"

\end_inset

 (see also section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Scripting-HOWTO"

\end_inset

; an example can be found in 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

contrib/example-scripts/check-mars-switchable.sh
\end_layout

\end_inset

).
 The network should be OK, and the amount of replication delay should be
 as low as possible.
 Otherwise, handover may take a very long time.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Best practice is to 
\series bold
prepare
\begin_inset Foot
status open

\begin_layout Plain Layout
Precondition for a plain 
\family typewriter
marsadm primary
\family default
 (without 
\family typewriter
systemd
\family default
) is that you are up, that means in attached and connected state (cf.
 
\family typewriter
marsadm up
\family default
), that you are no sync target anymore, and (only when 
\family typewriter
systemd
\family default
 isn't configured to automatically stop the application at the old site)
 that any old primary (in this case 
\family typewriter
A
\family default
) does not use its 
\family typewriter
/dev/mars/mydata
\family default
 device any longer, and that the network is healthy.
 If some (parts of) logfiles are not yet (fully) transferred to the new
 primary, you will need enough space on 
\family typewriter
/mars/
\family default
 at the target side.
 If one of the preconditions described in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Operation-of-the"

\end_inset

 is violated, 
\family typewriter
marsadm primary
\family default
 may refuse to start.
\end_layout

\begin_layout Plain Layout
These preconditions try to protect you from doing silly things, such as
 accidentally provoking a split brain error state.
 We try to avoid split brain as best as we can.
 Therefore, we distinguish between 
\emph on
intended
\emph default
 and 
\emph on
emergeny
\emph default
 switching.
 Intended switching will try to avoid split brain 
\emph on
as best as it can
\emph default
.
\end_layout

\end_inset

 a planned handover
\series default
 by the following steps:
\end_layout

\begin_layout Enumerate
Check the network and the replication lag.
 It should be low (a few hundred megabytes, or a low number of gigabytes
 – see also the rough time forecast shown by 
\family typewriter
marsadm view mydata
\family default
 when there is a larger replication delay, or directly access the forecast
 by 
\family typewriter
marsadm view-replinfo
\family default
).
\end_layout

\begin_layout Enumerate
Only when the 
\family typewriter
systemd
\family default
 method from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

 is 
\emph on
not
\emph default
 used: stop your application on hostA, then say on hostA:
\begin_inset Newline newline
\end_inset


\family typewriter
umount /dev/mars/mydata
\begin_inset Newline newline
\end_inset


\family default

\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

If you use the automatic handover method provided by systemd templates (see
 section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

), this step is 
\emph on
not needed
\emph default
.
\end_layout

\begin_layout Enumerate
Only when systemd templates are 
\emph on
not
\emph default
 used, and only for increased safety on hostA:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm wait-umount mydata
\begin_inset Newline newline
\end_inset


\family default
This will reduce the risk of 
\series bold
hanging umounts
\series default
 leading to long-lasting waits at the future primary hostB.
 Such problems will be detected earlier, so you have more possibilties for
 fixing them.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Also good practice: use 
\family typewriter
lsof /dev/mars/mydata
\family default
 before umount for even earlier detection of hanging processes.
\end_layout

\begin_layout Enumerate
Optionally, and only when the 
\family typewriter
systemd
\family default
 method from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

 is 
\emph on
not
\emph default
 used: on host 
\family typewriter
B
\family default
, wait until 
\family typewriter
marsadm view-diskstate mydata
\family default
 shows 
\family typewriter
UpToDate
\family default
.
 This way, you are gaining more control over the 
\emph on
duration
\emph default
 of the handover.
 In case of unexpected network problems, disk space problems, etc, you can
 script a compensation action like giving up much earlier, and restarting
 your application at the old primary hostA much earlier.
\end_layout

\begin_layout Enumerate
On hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm primary mydata
\family default

\begin_inset Newline newline
\end_inset

When combined with the 
\family typewriter
systemd
\family default
 method (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

), this will even automatically stop the application at hostA, wait for
 handover, and start the application at hostB.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

The most important difference to DRBD: don't use an intermediate 
\family typewriter
marsadm secondary mydata
\family default
 at hostA.
 Although it is possible, there are severeral 
\emph on
disadvantages
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\family typewriter
marsadm secondary
\family default
 is 
\emph on
discouraged
\emph default
 for several reasons.
 It tells the 
\emph on
whole cluster
\emph default
 that 
\emph on
nobody
\emph default
 is designated as primary anymore.
 
\emph on
All
\emph default
 nodes should go into secondary mode, globally.
 In the current version of MARS, the secondaries will no long fetch any
 logfiles, since in split brain situations they don't know which version
 is the 
\begin_inset Quotes eld
\end_inset

right
\begin_inset Quotes erd
\end_inset

 one.
 When a primary host is designated, this is the 
\begin_inset Quotes eld
\end_inset

right
\begin_inset Quotes erd
\end_inset

 one by definition.
 Syncing is also not possible when there is no designated primary.
 When the device 
\family typewriter
/dev/mars/mydata
\family default
 is in use somewhere, it will remain in 
\emph on
actual
\emph default
 primary mode during that time, and the secondaries will sync therefrom.
 As soon as the local 
\family typewriter
/dev/mars/mydata
\family default
 is released, the node will 
\emph on
actually
\emph default
 go into secondary mode if it is no longer designated as primary.
\end_layout

\end_inset

 from losing the primary state.
 In case of an unexpected crash at the wrong moment, nobody might know anymore
 where the primary was running before.
 Best practice is to always switch 
\emph on
directly
\emph default
 from the old primary hostA to the new primary hostB.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 If you need the local device 
\family typewriter
/dev/mars/mydata
\family default
 to disappear 
\emph on
everywhere
\emph default
 in the whole cluster, you don't need the discouraged 
\family typewriter
marsadm secondary
\family default
 command.
 
\family typewriter
marsadm detach
\family default
 or 
\family typewriter
marsadm down
\family default
 can do it also, without destroying knowledge about the former designated
 primary.
 There is only one use case where 
\family typewriter
marsadm secondary
\family default
 is really needed: final destruction of a resource before 
\family typewriter
marsadm delete-resource
\family default
 is executed.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

In contrast to DRBD, MARS remembers the designated primary, even when your
 system crashes and reboots.
 With DRBD, you typically will have to re-setup the DRBD roles with (scripted)
 commands like 
\family typewriter
drbdadm up 
\begin_inset Formula $\ldots$
\end_inset

; drbdadm primary 
\begin_inset Formula $\ldots$
\end_inset


\family default
.
 Instead, MARS will 
\series bold
automatically resume
\series default
 its former roles just by saying 
\family typewriter
modprobe mars
\family default
.
 When combined with a proper 
\family typewriter
systemd
\family default
 setup (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

), this will even automatically re-start your application after the crash.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Another fundamental difference to DRBD: when the network is healthy, there
 can only exist 
\emph on
one
\emph default
 designated primary at a time.
 By saying 
\family typewriter
marsadm primary mydata
\family default
 on host
\family typewriter
B
\family default
, 
\series bold
all other
\series default
 hosts (including 
\family typewriter
hostA
\family default
) will 
\series bold
automatically go into secondary role
\series default
 after a while.
 You don't need to tell them explicitly, because MARS is automatically propagati
ng the information for you.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 A 
\emph on
very rough
\emph default
 estimation of the time to become 
\family typewriter
UpToDate
\family default
 is displayed by 
\family typewriter
marsadm view mydata
\family default
 or other macros (e.g.
 
\family typewriter
view-replinfo
\family default
).
 However, on very flaky networks, the estimation may be flickering.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Planned handover is refused 
\emph on
by default
\emph default
 when some sync is running somewhere, even at a third hostC.
 By adding the option 
\family typewriter
--ignore-sync
\family default
, you are no longer protected by this 
\emph on
safety measure
\emph default
, and you are willing to accept that any already running sync at any hostC
 or hostD will restart from point 0, in order to ensure consistency.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Tip: newer versions of 
\family typewriter
mars
\family default
 + 
\family typewriter
marsadm
\family default
 are supporting the option 
\family typewriter
--parallel
\family default
 combined with 
\family typewriter
all
\family default
, e.g.
 
\family typewriter
marsadm primary all --parallel
\family default
.
 Instead of waiting until 
\emph on
all
\emph default
 the resources have left the primary role at the old primary (barrier synchoniza
tion), the handover speed of each resource is treated 
\emph on
individually
\emph default
.
 Slow resources will no longer retard fast ones, minimizing total downtime.
 However, check that your cluster manager can deal with a rather high parallelis
m degree.
 At the moment, the 
\family typewriter
systemd
\family default
 interface is not yet prepared for this.
\end_layout

\begin_layout Subsection
Forced Switching
\begin_inset CommandInset label
LatexCommand label
name "subsec:Forced-Switching"

\end_inset


\end_layout

\begin_layout Standard
In case of an incident, the connection to the old primary hostA may be lost
 for several reasons.
 Then, at hostB, we just don't know anything about its 
\emph on
current
\emph default
 state (which may deviate from its 
\emph on
last known
\emph default
 state).
 The following command sequence will skip many checks (essentially you just
 need to be attached and you must not be a current sync target) and tell
 hostB to become primary forcefully:
\end_layout

\begin_layout Enumerate
On hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm pause-fetch mydata
\family default

\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 notice that this is similar to 
\family typewriter
drbdadm disconnect mydata
\family default
 as you might be used from DRBD.
 For better compatibility with DRBD, you may use the alternate syntax 
\family typewriter
marsadm disconnect mydata
\family default
 instead.
 However, there is a subtle difference to DRBD: DRBD will drop 
\emph on
both
\emph default
 sides of its single bi-directional connection and no longer try to re-connect
 from any of both sides.
 In contrast, 
\family typewriter
marsadm pause-fetch
\family default
 is equivalent to 
\family typewriter
pause-fetch-local
\family default
, which instructs only the 
\emph on
local
\emph default
 host to stop fetching logfiles.
 Other members of the cluster, including the former primary, are 
\emph on
not
\emph default
 instructed to do so.
 They may continue fetching logfiles over their own private TCP connections,
 potentially using many connections in parallel, potentially distributed
 over multiple routes, and potentially even from any 
\emph on
other
\emph default
 member of the resource, if they think they can get the data from there.
 In order to instruct
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that not all such instructions may arrive at all sites when the network
 is interrupted (or extremely slow).
\end_layout

\end_inset

 
\emph on
all
\emph default
 members of the resource to stop fetching logfiles, you may use 
\family typewriter
marsadm pause-fetch-global mydata
\family default
 instead (cf section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Operation-of-the"

\end_inset

).
\end_layout

\end_deeper
\begin_layout Enumerate
On hostB:
\family typewriter

\begin_inset Newline newline
\end_inset

marsadm primary --force mydata
\family default

\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 this is the forceful failover.
 Depending on the current replication lag, you may lose some data.
 Use 
\family typewriter
--force
\family default
 only if you know what you are doing!
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 When 
\family typewriter
systemd
\family default
 is configured properly (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

), your application will start automatically at the new primary site.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 when the replication network is interrupted while the old primary hostA
 continues
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice: in certain network outage scenarios, you may not be able to remotely
 login to the console and to check whether a server is running.
 Therefore it may happen that you erronously think hostA is dead, while
 in reality it continues running.
 Even if you would know it, you might not be able to remotely kill it in
 a STONITH-like manner.
\end_layout

\end_inset

 running, it cannot know that hostB is the new designated primary.
 Therefore hostA will continue running by default.
 This means that your application will run twice! Only when the metadata
 exchange is working again (by default on port 7777), the old hostA will
 be automatically shut down by its local 
\family typewriter
systemd
\family default
 configuration, when configured properly (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

).
 In difference to the 
\emph on
planned
\emph default
 handover from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Intended-Switching"

\end_inset

, this may happen much later.
 In case of very long-last network outages, it may take even days or weeks.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

Running both sites in parallel for a long time may seriously damage your
 business.
 Ensure that any 
\series bold
customer traffic
\series default
 cannot go to the old site! Be sure to configure your BGP in a proper way,
 such that 
\emph on
only
\emph default
, and 
\emph on
only
\emph default
 the new site will receive any customer traffic from both inside and outside
 networks, like the internet.
\end_layout

\end_deeper
\begin_layout Enumerate
For safety on hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm resume-fetch mydata
\family default

\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Standard
The new primary would not really need this, because primaries are producing
 their own logfiles without need for fetching.
 This is only to undo the previous 
\family typewriter
pause-fetch
\family default
, in order to avoid future surprises when the new primary will somewhen
 change to secondary mode again (in the far-distant future), and you have
 forgotten to remember the fact that fetching had been switched off.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Newer marsadm versions, starting from 
\family typewriter
mars0.1astable113
\family default
, do not need this step anymore.
 After successful activation of 
\family typewriter
/dev/mars/mydata
\family default
, the equivalent of 
\family typewriter
marsadm up mydata
\family default
 is executed automatically.
\end_layout

\end_deeper
\begin_layout Standard
When using 
\family typewriter
--force
\family default
, many precondition checks and other internal checks are skipped, in particular
 the internal handover protocol for split brain avoidance.
\end_layout

\begin_layout Standard
In general, use of 
\family typewriter
--force
\family default
 is 
\emph on
likely
\emph default
 to 
\series bold
provoke a split brain
\series default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 
\series bold
Split brain
\series default
 is always an 
\series bold
erroneous state
\series default
 which should be never entered without reason! Once you have entered it
 accidentally, you 
\series bold
must
\series default
 resolve it ASAP (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Split-Brain-Resolution"

\end_inset

), otherwise you cannot operate your resource in the long term.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 In case of 
\emph on
connection loss
\emph default
 (e.g.
 networking problems / network partitions), you may not be able to reliably
 detect whether a split brain has actually occured, or not.
\end_layout

\begin_layout Paragraph
Some Background (may be skipped)
\end_layout

\begin_layout Standard
In contrast to DRBD, split brain situations are handled differently by MARS
 .
 When two primaries are accidentally active at the same time, each of them
 writes into different logfiles 
\family typewriter
/mars/resource-mydata/log-000000001-hostA
\family default
 and 
\family typewriter
/mars/resource-mydata/log-000000001-hostB
\family default
 where the 
\emph on
origin
\emph default
 host is always recorded in the filename.
 Therefore, both nodes 
\emph on
can theoretically
\emph default
 run in primary mode independently from each other, at least for some time.
 They 
\emph on
might
\emph default
 even 
\family typewriter
log-rotate
\family default
 independently from each other.
 However, this is really no good idea.
 The replication to third nodes will likely get stuck, and your 
\family typewriter
/mars/
\family default
 filesystem(s) will eventually run out of space.
 Any further secondary node (when having 
\begin_inset Formula $k>2$
\end_inset

 replicas) will certainly get into serious problems: it simply does not
 know which split-brain version it should follow.
 Therefore, you will certainly loose the actuality of your redundancy.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Split brain situations are detected 
\emph on
passively
\emph default
 by secondaries.
 Whenever a secondary detects that somewhere a split brain has happend,
 it refuses to replay any logfiles behind the split point (and also to fetch
 them when possible), or anywhere where something appears suspect or ambiguous.
 This tries to keep its local disk state always being 
\begin_inset Quotes eld
\end_inset

as consistent as possible
\begin_inset Quotes erd
\end_inset

, but outdated with respect to any of the split brain versions.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 
\family typewriter
marsadm primary –force
\family default
 is rejected in newer marsadm versions
\begin_inset Foot
status open

\begin_layout Plain Layout
Beware: older versions before 
\family typewriter
mars0.1stable52
\family default
 did deliberately skip this check because a few years ago somebody at 1&1
 placed a 
\emph on
requirement
\emph default
 on this.
 Fortunately, the requirement now has gone, so a safee behaviour could be
 implemented.
 The new behaviour is for your safety, to prevent you from doing 
\begin_inset Quotes eld
\end_inset

silly
\begin_inset Quotes erd
\end_inset

 things in case you are under pressure during an incident (try to safeguard
 human error as best as possible).
\end_layout

\end_inset

 if your replica is a current sync target.
 This is not a bug: it should prevent you from forcing an inconsistent replica
 into primary mode, which will 
\emph on
certainly
\emph default
 lead to inconsistent data.
 However, in extremely rare cases of severe damage of 
\emph on
all
\emph default
 of your replicas, you may be desperate.
 Only in such a rare case, and only then, you might decide to force any
 of your replicas (e.g.
 based on their last sync progress bar) into primary role although none
 of the re-syncs had finished before.
 In such a case, and only if you really know what you are doing, you may
 use 
\family typewriter
marsadm fake-sync
\family default
 to first mark your inconsistent replica as UpToDate (which is a 
\series bold
lie
\series default
) and then force it to primary as explained above.
 Afterwards, you will certainly need 
\family typewriter
fsck
\family default
 or another type of repair before you can restart your application.
 Good luck! And don't forget to check the size of 
\family typewriter
lost+found
\family default
 afterwards.
 This is really your 
\emph on
very last
\emph default
 chance if nothing else had succeeded before.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Tip: newer versions of 
\family typewriter
mars
\family default
 + 
\family typewriter
marsadm
\family default
 are supporting the option 
\family typewriter
--parallel
\family default
 combined with 
\family typewriter
all
\family default
, e.g.
 
\family typewriter
marsadm primary --force all --parallel
\family default
.
 This may potentially speed up startup.
 However, check that your cluster manager can deal with a rather high parallelis
m degree.
 At the moment, the 
\family typewriter
systemd
\family default
 interface is not yet prepared for this.
\end_layout

\begin_layout Section
Split Brain Resolution
\begin_inset CommandInset label
LatexCommand label
name "subsec:Split-Brain-Resolution"

\end_inset


\end_layout

\begin_layout Standard
Split brain can naturally occur during a long-lasting network outage (aka
 network partition) when you (forcefully) switch primaries inbetween, or
 due to final loss of your old primary node (fatal node crash) when not
 all logfile data had been transferred immediately before the final crash.
\end_layout

\begin_layout Standard
In general, 
\series bold
split brain is unavoidable
\series default
 in 
\emph on
any
\emph default
 distributed system, even if you use a passive crossover cable with DRBD,
 and when the crossover cable fails at the wrong moment.
 Please search Wikipedia for the CAP theorem, or read the corresponding
 section in 
\family typewriter
mars-architecture-guide.pdf
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Remember that split brain is an 
\series bold
erroneous state
\series default
 which must be resolved as soon as possible!
\end_layout

\begin_layout Standard
Whenever split brain occurs for whatever reason, you have two choices for
 resolution: either destroy one of your versions, or retain it under a different
 resource name.
\end_layout

\begin_layout Standard
In any of both cases, do the following steps ASAP:
\end_layout

\begin_layout Enumerate

\series bold
Manually
\series default
 check which (surviving) version is the 
\begin_inset Quotes eld
\end_inset

right
\begin_inset Quotes erd
\end_inset

 one.
 Any error is up to you: destroying the wrong version is 
\emph on
your
\emph default
 fault.
\begin_inset Newline newline
\end_inset

Newer versions of 
\family typewriter
marsadm view
\family default
 are supporting your decision by telling you the amount of logfile data
 you would destroy when destroying a certain version of a certain host.
 Typically, the smaller version is a candidate for destruction.
 However, there are situations where this may be wrong, such as amok-running
 applications running like endless loops, or Spammer attacks against databases,
 or similar.
 MARS cannot know about this.
\end_layout

\begin_layout Enumerate
If you did not already switch your primary to the final destination determined
 in the previous step, do it now (see description in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Forced-Switching"

\end_inset

).
 Don't use an intermediate 
\family typewriter
marsadm secondary
\family default
 command (as known from DRBD): 
\emph on
directly
\emph default
 switch to the new designated primary!
\end_layout

\begin_layout Enumerate
Unless 
\family typewriter
systemd
\family default
 is configured properly (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:systemd-Templates"

\end_inset

), do the following manually: on each non-right version (which you don't
 want to retain) which had been primary before, umount your 
\family typewriter
/dev/mars/mydata
\family default
 or otherwise stop using it (e.g.
 stop iSCSI or other users of the device).
 Wait until each of them has actually left primary state and until their
 local logfile(s) have been fully written back to the underlying disk.
\end_layout

\begin_layout Enumerate
Wait until the network works again.
 All your (surviving) cluster nodes 
\emph on
must
\emph default

\begin_inset Foot
status open

\begin_layout Plain Layout
If you are a MARS expert and you really know what you are doing (in particular,
 you can anticipate the effects of the Lamport clock and of the symlink
 update protocol including the 
\begin_inset Quotes eld
\end_inset

eventually consistent
\begin_inset Quotes erd
\end_inset

 behaviour including any not-yet-consistent intermediate states), you may
 deviate from this requirement, at your own risk.
\end_layout

\end_inset

 be able to communicate with each other.
 If this is not possible, or if it would take too long, fall back to one
 of the method described in appendix ......
 ......
 section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Final-Destroy-of"

\end_inset

, but do this only when necessary.
\end_layout

\begin_layout Standard
The next steps are different for different use cases:
\end_layout

\begin_layout Paragraph
Destroying a Wrong Split Brain Version
\end_layout

\begin_layout Standard
Continue with the following steps, each on those cluster node(s) where you
 do not want to retain its split-brain version.
 In preference, start with the old 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Quotes erd
\end_inset

 primaries first (see advice at the end of this section):
\end_layout

\begin_layout Enumerate-Resume
On 
\series bold
all
\series default
 affected secondary nodes hostX where SPLIT BRAIN is reported:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm invalidate mydata
\family default

\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
setcounter{enumi}{4}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\noindent
Typically, no split brain is reported anymore after that (via 
\family typewriter
marsadm view all
\family default
), and you are done.
\end_layout

\begin_layout Standard
In rare cases (when 
\family typewriter
/mars
\family default
 is almost full somewhere, or when emergency mode has occured somewhere),
 you may need to run 
\family typewriter
marsadm cron
\family default
 at the primary host, and to repeat 
\family typewriter
marsadm invalidate
\family default
 on any SPLIT BRAIN host.
 In extremely rare of overloaded nodes, you may need to repeat this several
 times.
\end_layout

\begin_layout Standard
In very rare cases, when things are screwed up very heavily (e.g.
 a partly destroyed 
\family typewriter
/mars/
\family default
 partition), you may try an alternate method described in appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Alternative-Methods-for"

\end_inset

.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Check that state 
\family typewriter
Orphan
\family default
 is left after a while.
 Notice that 
\family typewriter
invalidate
\family default
 is only 
\emph on
restarting
\emph default
 an existing replica, but does not wait for its completion.
\end_layout

\begin_layout Paragraph
Retaining a Split Brain Version (optionally, typically not needed, may be
 skipped)
\begin_inset CommandInset label
LatexCommand label
name "par:Retaining-a-Split"

\end_inset


\end_layout

\begin_layout Standard
On those cluster nodes where you want to retain some SPLIT BRAIN version
 (e.g.
 for inspection or debugging purposes, or as a kind of 
\begin_inset Quotes eld
\end_inset

emergengy backup
\begin_inset Quotes erd
\end_inset

):
\end_layout

\begin_layout Enumerate-Resume

\family typewriter
marsadm down mydata
\end_layout

\begin_layout Enumerate-Resume

\family typewriter
marsadm leave-resource mydata
\end_layout

\begin_layout Enumerate-Resume
After having done this on 
\emph on
all
\emph default
 those cluster nodes, check that the split brain is gone (e.g.
 by saying 
\family typewriter
marsadm view mydata
\family default
 at the primary).
 In very rare cases, you might also need a 
\family typewriter
log-purge-all
\family default
 at the primary (see page 
\begin_inset CommandInset ref
LatexCommand pageref
reference "log-purge-all$res"

\end_inset

).
\end_layout

\begin_layout Enumerate-Resume
Rename the underlying local 
\family typewriter
/dev/lv/mydata
\family default
 is into something like 
\family typewriter
/dev/lv/mydata
\family default
-backup.
 For example:
\begin_inset Newline newline
\end_inset


\family typewriter
lvrename lv mydata mydata-backup
\family default

\begin_inset Newline newline
\end_inset

For details, see 
\family typewriter
man lvrename
\family default
.
 This is 
\emph on
extremely
\emph default
 recommended to avoid confusion with the old resource name.
\end_layout

\begin_layout Enumerate-Resume
For safety: check that each underlying local disk 
\family typewriter
/dev/lv/mydata-backup
\family default
 is really usable afterwards, e.g.
 by test-mounting it (and 
\family typewriter
fsck
\family default
 when needed).
 If all is OK, don't forget to umount it before proceeding with the next
 step.
\end_layout

\begin_layout Enumerate-Resume
Finally, you may either delete your backup somewhen via 
\family typewriter
lvremove
\family default
, or you may create a completely new MARS resource out of it, but 
\emph on
under a different name
\emph default
.
 See description in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:Creating-and-Maintaining"

\end_inset

.
\end_layout

\begin_layout Enumerate-Resume
Optionally, if you have enough disk space (check with 
\family typewriter
vgs
\family default
): re-create your replica by freshly creating a new 
\family typewriter
/dev/vg/mydata
\family default
 with the right size, and 
\family typewriter
marsadm join-resource mydata /dev/vg/mydata
\family default
.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Generally:
\series bold
 best practice
\series default
 is to always keep your LV names equal to your MARS resource names.
 This can avoid a 
\emph on
lot
\emph default
 of unnecessary confusion.
\end_layout

\begin_layout Paragraph
Keeping a Good Version (typically no actions needed)
\end_layout

\begin_layout Standard
When you had a secondary which did not participate in the split brain, but
 just got confused and therefore stopped replaying logfiles immediately
 before the split-brain point, it will typically
\begin_inset Foot
status open

\begin_layout Plain Layout
In general, such a 
\begin_inset Quotes eld
\end_inset

good
\begin_inset Quotes erd
\end_inset

 behaviour cannot be guaranteed for all secondaries.
 Race conditions in complex networks may asynchronously transfer 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Quotes erd
\end_inset

 logfile data to a secondary much earlier than conflicting 
\begin_inset Quotes eld
\end_inset

good
\begin_inset Quotes erd
\end_inset

 logfile data which will be marked 
\begin_inset Quotes eld
\end_inset

good
\begin_inset Quotes erd
\end_inset

 only in the 
\emph on
future.

\emph default
 It is impossible to predict this in advance.
\end_layout

\end_inset

 resume replay after the SPLIT BRAIN has been resolved at the other nodes.
 Then you don't need to do any action for it.
\end_layout

\begin_layout Standard
When all SPLIT BRAIN versions have disappeared from the cluster (by 
\family typewriter
invalidate
\family default
 or 
\family typewriter
leave-resource
\family default
 as described before), the confusion should be over, and the secondary should
 automatically resume tracking of the new unique version.
\end_layout

\begin_layout Standard
Please check that 
\emph on
all
\emph default
 of your secondaries are no longer stuck.
 You need to execute split brain resolution only for 
\emph on
stuck
\emph default
 nodes.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Hint / advice for 
\begin_inset Formula $k>2$
\end_inset

 replicas: it is a good idea to start split brain resolution 
\emph on
first
\emph default
 with those (few) nodes which had been (accidentally) primary before, but
 are not the new designated primary.
 Usually, you had 2 primaries during split brain, so this will apply only
 to 
\emph on
one
\emph default
 of them.
 Leave the other one intact, by not umounting 
\family typewriter
/dev/mars/mydata
\family default
 at all, and keeping your applications running.
 Even during emergency mode, see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Emergency-Mode"

\end_inset

.
 
\emph on
First
\emph default
 resolve the problem of the 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Quotes erd
\end_inset

 primary(s) via 
\family typewriter
invalidate
\family default
 or 
\family typewriter
leave-resource
\family default
.
 Wait for a short while.
 Then check the rest of your secondaries, whether they now are already following
 the new (unique) primary, and finally check whether the split brain warning
 reported by 
\family typewriter
marsadm view all
\family default
 is gone everywhere.
 This way, you can often skip unnecessary invalidations of replicas.
\end_layout

\begin_layout Section
Final Destruction of a Damaged Node
\begin_inset CommandInset label
LatexCommand label
name "subsec:Final-Destroy-of"

\end_inset


\end_layout

\begin_layout Standard
When a node has eventually died (e.g.
 defective hardware), 
\series bold
do not forget
\series default

\begin_inset Foot
status open

\begin_layout Plain Layout
If you forget this, 
\family typewriter
/mars
\family default
 will fill up forever.
 Finally, emergency mode will be triggered.
\end_layout

\end_inset

 the following steps ASAP:
\end_layout

\begin_layout Enumerate

\emph on
Physically
\emph default
 remove the dead node from your network.
 Unplug all network cables! Failing to do so might provoke a disaster in
 case it somehow resurrects in an uncontrolled manner, such as a partly-damaged
 
\family typewriter
/mars/
\family default
 filesystem, a half-defective kernel, RAM / kernel memory corruption, disk
 corruption, or whatever.
 Although MARS has some provisions like md5 checksums in its transaction
 logfiles: don't risk any 
\series bold
unpredictable behaviour
\series default
!
\end_layout

\begin_layout Enumerate

\series bold
Manually
\series default
 check which of the surviving versions will be the 
\begin_inset Quotes eld
\end_inset

right
\begin_inset Quotes erd
\end_inset

 one.
 Any human error is up to you: resurrecting an unnecessarily old / outdated
 version and/or decommissioning the productive primary server will be 
\emph on
your
\emph default
 fault.
\end_layout

\begin_layout Enumerate
If you did not already switch your primary to the final destination determined
 in the previous step, do it now (see description in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Forced-Switching"

\end_inset

).
\end_layout

\begin_layout Enumerate
On a surviving node, give the following commands:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Enumerate

\family typewriter
marsadm --host=your-damaged-host down mydata --force
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm --host=your-damaged-host leave-resource mydata --force
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Check for misspellings, in particular the hostname of the dead node, and
 check the command syntax before typing return! Otherwise, you may forcefully
 destroy the wrong
\begin_inset Foot
status open

\begin_layout Plain Layout
That said, MARS appears to be rather tolerant of human errors.
 As long as your 
\family typewriter
/dev/vg/mydata
\family default
 is not removed at LVM level, you have a chance for recovery.
 Once a sysadmin destroyed a whole cluster by accident, including all of
 its resources, and while it was continuously running in primary role.
 Even transaction logging did continue on some orphan logfiles, but 
\family typewriter
/mars
\family default
 was filling up 
\begin_inset Quotes eld
\end_inset

unexpectedly
\begin_inset Quotes erd
\end_inset

.
 Fortunately, this behaviour led to a monitoring alert and to detection
 of the problem.
 It was early enough for a correction without causing any extraordinary
 customer downtime outside of accepted SLAs, and no data loss at all.
\end_layout

\end_inset

 node!
\end_layout

\end_deeper
\begin_layout Enumerate
Repeat the same with 
\emph on
all
\emph default
 resources which were formerly present at 
\family typewriter
your-damaged-host
\family default
.
\end_layout

\begin_layout Enumerate
Finally, say
\family typewriter

\begin_inset Newline newline
\end_inset

marsadm --host=your-damaged-host leave-cluster --force
\end_layout

\begin_layout Standard
Now all your surviving nodes should 
\emph on
believe
\emph default
 that the old node 
\family typewriter
your-damaged-host
\family default
 does no longer exist, and that it does no longer participate in any resource.
 For safety, check this via 
\family typewriter
marsadm view
\family default
 everywhere.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Even if your dead node comes to life again in some way: always ensure that
 the mars kernel module cannot run any more on such a zombie server.
 
\emph on
Never
\emph default
 do a 
\family typewriter
modprobe mars
\family default
 on a node marked as dead this way!
\end_layout

\begin_layout Standard
Further instructions for complicated cases of destruction are in appendix
 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Alternative-De--and"

\end_inset

 and 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Cleanup-in-case"

\end_inset

.
\end_layout

\begin_layout Section
Online Resizing during Operation
\begin_inset CommandInset label
LatexCommand label
name "sec:Online-Resizing-during"

\end_inset


\end_layout

\begin_layout Standard
You need LVM or some other means of increasing the physical size of your
 disk (e.g.
 via firmware of some RAID controllers).
 The network must be healthy.
 Do the following steps:
\end_layout

\begin_layout Enumerate
Increase your local disks (usually 
\family typewriter
/dev/vg/mydata
\family default
) 
\emph on
everywhere
\emph default
 in the whole cluster.
 In order to avoid wasting space, increase them 
\emph on
uniformly
\emph default
 to the same size (when possible).
 For example, on both hostA and hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
lvresize -L +100G /dev/vg/mydata
\end_layout

\begin_layout Enumerate
For safety, say on both hostA and hostB
\family typewriter

\begin_inset Newline newline
\end_inset

marsadm up mydata
\end_layout

\begin_layout Enumerate
For safety, say on both hostA and hostB
\family typewriter

\begin_inset Newline newline
\end_inset

marsadm wait-cluster
\end_layout

\begin_layout Enumerate
Only at the primary hostA:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm resize mydata
\end_layout

\begin_layout Enumerate
A 
\emph on
partial
\emph default
 fast full-sync will start at hostB.
 Only the 
\emph on
new
\emph default
 portion of the block device will be synced.
 Check that sync is running, or has already finished.
\end_layout

\begin_layout Enumerate
If you have intermediate layers such as iSCSI, you may need some 
\family typewriter
iscsiadm
\family default
 update or other commands.
\end_layout

\begin_layout Enumerate
Generally not needed, only for 
\emph on
extreme
\emph default
 safety / paranoia: you may wait until the partial sync has finished.
 This is not really needed, but it may 
\emph on
slightly(!)
\emph default
 reduce the risk in case of an unplanned incident at the primary side.
 If you execute the last step before the sync has finished, some data might
 have been already written to the new portions of the underlying LV.
 These writes will be written to the transaction log, and will be replicated.
 Thus there is no real danger, and your secondary will be 
\series bold
logically consistent
\series default
 in reality, although it will be 
\emph on
reported
\emph default
 as 
\family typewriter
InConsistent
\family default
 by marsadm in the meantime.
 If you want to avoid confusion about state 
\family typewriter
Consistent
\family default
, and/or if you want extremely high protection against damaged logfiles
 by hardware defects at the wrong moment (although the transaction logfiles
 are already protected by md5 checksums), and/or if you are paranoid, it
 may be helpful to wait until the sync has finished.
 Normally, this is not needed, and you may immediately proceed to the last
 step:
\end_layout

\begin_layout Enumerate
Now you may increase your filesystem.
 This is specific for the filesystem type and documented elsewhere.
 Some filesystems are able to increase their size while they are mounted
 and while the applications are running on top of them, but others cannot
 do this.
 For example, an xfs online resize during operation can be triggered at
 primary hostA where 
\family typewriter
/dev/mars/mydata
\family default
 is currently mounted:
\begin_inset Newline newline
\end_inset


\family typewriter
xfs_growfs /mountpoint/of/mydata
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Hint: in general, the sync of the new portions is not really needed, because
 the new junk data just does not care at filesystem level.
 If you are not paranoid and if you know what you are doing, you may use
 
\family typewriter
marsadm fake-sync mydata
\family default
 to abort unnecessary network traffic.
\end_layout

\begin_layout Section
Defending Overflow of 
\family typewriter
/mars/
\begin_inset CommandInset label
LatexCommand label
name "sec:Defending-Overflow"

\end_inset


\end_layout

\begin_layout Standard
This section describes an important difference to DRBD.
 The metadata of DRBD is allocated 
\emph on
statically
\emph default
 at 
\emph on
creation
\emph default
 
\emph on
time
\emph default
 of the resource.
 In contrast, the MARS transaction logfiles are allocated 
\emph on
dynamically
\emph default
 at 
\emph on
runtime
\emph default
.
\end_layout

\begin_layout Standard
This leads to a potential risk from the perspective of a sysadmin: what
 happens if the 
\family typewriter
/mars/
\family default
 filesystem runs out of space?
\end_layout

\begin_layout Standard
In practice, no harm will occur to your data.
 MARS will automatically go into the so-called emergency mode.
 Resolution of emergency mode is very similar to resolution of split brain
 (section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Resolution-of-Split"
plural "false"
caps "false"
noprefix "false"

\end_inset

): at all of your secondaries, type (repeatedly)
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm invalidate all
\end_layout

\begin_layout Standard
This is all you need to know.
 If you are impatient, you may now skip the rest of this section.
\end_layout

\begin_layout Standard
For some background explanations, keep reading on.
\end_layout

\begin_layout Standard
Overflow and its treatment is 
\emph on
unavoidable
\emph default
 for long-distance replication.
 If you want a system which can survive long-lasting network outages, while
 keeping your replicas consistent as long as possible (called 
\series bold
anytime consistency
\series default
), then you 
\emph on
need
\emph default
 dynamic storage.
 It is 
\emph on
impossible
\emph default
 to solve with static pre-allocated memory
\begin_inset Foot
status open

\begin_layout Plain Layout
The bitmaps used by DRBD cannot preserve the 
\emph on
order
\emph default
 of write operations.
 They cannot do that, because their space is 
\begin_inset Formula $O(k)$
\end_inset

 for some constant 
\begin_inset Formula $k$
\end_inset

.
 In contrast, MARS preserves the order.
 Preserving the order as such (even when only 
\emph on
facts
\emph default
 about the order were recorded without recording the actual data contents)
 requires 
\begin_inset Formula $O(n)$
\end_inset

 space where 
\begin_inset Formula $n$
\end_inset

 is infinitely growing over time.
\end_layout

\end_inset

.
 A true solution would need 
\emph on
infinite memory
\emph default
.
 But suchalike does not exist on earth.
\end_layout

\begin_layout Standard
It would be an even worse idea to statically pre-allocate a lot of space
 for 
\emph on
each
\emph default
 of your resources.
 The latter would waste a lot of space, because some resources will likely
 fill much more quickly than others.
 MARS deals with this by using a 
\emph on
common
\emph default
 filesystem /mars which is 
\emph on
shared
\emph default
 by the transaction logs of 
\emph on
all
\emph default
 of your resources.
\end_layout

\begin_layout Standard
Although the size of /mars is statically allocated at cluster generation
 time, there is a workaround for the problem.
 When 
\family typewriter
/mars
\family default
 fills up during a network outage, and you have some spare space on your
 VG, and when the network outage will be repaired shortly, you may decide
 to dynamically extend /mars during operation.
\end_layout

\begin_layout Standard
Because of these fundamental differences, DRBD and MARS have different applicati
on areas.
 If you just want a simple system for mirroring your data over short distances
 via passive
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice: newer generation 10GBit technologies like SFP+ are no longer passive.
 They involve some active chips, which may fail independently from your
 servers.
 In case of a failure, the CAP theorem property P is violated, and you only
 have the choice between C and A.
 For details, see 
\family typewriter
mars-architecture-guide.pdf
\family default
.
\end_layout

\end_inset

 crossover cable, and when failures of the crossover cables are very unlikely,
 DRBD will be a suitable choice.
 However, if you need to replicate over longer distances, or if you need
 higher levels of reliability even when multiple failures may accumulate
 (such as network loss during a 
\emph on
re
\emph default
sync of DRBD), the transaction logs of MARS can solve it, but at some 
\emph on
cost
\emph default
.
\end_layout

\begin_layout Subsection
Countermeasures against overflow
\end_layout

\begin_layout Subsubsection
Dimensioning of 
\family typewriter
/mars/
\begin_inset CommandInset label
LatexCommand label
name "subsec:Dimensioning-of-/mars/"

\end_inset


\end_layout

\begin_layout Standard
The first (and most important) measure against overflow of 
\family typewriter
/mars/
\family default
 is simply to dimension it large enough to survive longer-lasting problems,
 preferably one weekend.
\end_layout

\begin_layout Standard
Recommended size is at least one dedicated disk, residing at a hardware
 RAID controller with BBU (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Preparation:-What-you"

\end_inset

).
 During normal operation, that size is needed only for a small fraction,
 typically a few percent or even less than one percent.
 However, it is your 
\series bold
safety margin
\series default
.
 Keep it high enough!
\end_layout

\begin_layout Subsubsection
Monitoring
\end_layout

\begin_layout Standard
The next (equally important) measure is 
\series bold
monitoring in userspace
\series default
.
\end_layout

\begin_layout Standard
Following is a list of countermeasures both in userspace and in kernelspace,
 in the order of 
\begin_inset Quotes eld
\end_inset

defensive walling
\begin_inset Quotes erd
\end_inset

:
\end_layout

\begin_layout Enumerate
Regular userspace monitoring must throw an INFO if a certain freespace limit
 
\begin_inset Formula $l_{1}$
\end_inset

 of 
\family typewriter
/mars/
\family default
 is undershot.
 Typical values for 
\begin_inset Formula $l_{1}$
\end_inset

 are 30%.
 Typical actions are automated calls of 
\family typewriter
marsadm cron
\family default
 (or 
\family typewriter
marsadm log-rotate all
\family default
 followed by 
\family typewriter
marsadm log-delete-all all
\family default
).
 You have to implement that yourself in sysadmin space.
\end_layout

\begin_layout Enumerate
Regular userspace monitoring must throw a WARNING if a certain freespace
 limit 
\begin_inset Formula $l_{2}$
\end_inset

 of 
\family typewriter
/mars/
\family default
 is undershot.
 Typical values for 
\begin_inset Formula $l_{2}$
\end_inset

 are 20%.
 Typical actions are (in addition to 
\family typewriter
log-rotate
\family default
 and 
\family typewriter
log-delete-all
\family default
) alarming human supervisors via SMS and/or further stronger automated actions.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Frequently large space is occupied by files stemming from debugging output,
 or from other programs or processes.
 A hot candidate is 
\begin_inset Quotes eld
\end_inset

forgotten
\begin_inset Quotes erd
\end_inset

 removal of debugging output to 
\family typewriter
/mars/
\family default
.
 Sometimes, an 
\family typewriter
rm -rf $(find /mars/ -name 
\begin_inset Quotes eld
\end_inset

*.log
\begin_inset Quotes erd
\end_inset

)
\family default
 can work miracles.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Another source of space hogging is a 
\begin_inset Quotes eld
\end_inset

forgotten
\begin_inset Quotes erd
\end_inset

 
\family typewriter
pause-sync
\family default
 or 
\family typewriter
disconnect
\family default
.
 Therefore, a simple 
\family typewriter
marsadm up all
\family default
 may also work miracles (if you didn't want to freeze some mirror deliberately).
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

If you just wanted to freeze a mirror at an outdated state for a very long
 time, you simply 
\emph on
cannot
\emph default
 do that without causing infinite growth of space consumption in 
\family typewriter
/mars/
\family default
.
 Therefore, a 
\family typewriter
marsadm leave-resource $res
\family default
 at 
\emph on
exactly that(!)
\emph default
 secondary site where the mirror is frozen, can also work miracles.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Hint: you can / should start some of these measures even earlier at the
 INFO level (see item 1), or even earlier.
\end_layout

\begin_layout Enumerate
Regular userspace monitoring must throw an ERROR if a certain freespace
 limit 
\begin_inset Formula $l_{3}$
\end_inset

 of 
\family typewriter
/mars/
\family default
 is undershot.
 Typical values for 
\begin_inset Formula $l_{3}$
\end_inset

 are 10%.
 Typical actions are alarming the CEO via SMS and/or even stronger automated
 actions.
 For example, you may choose to automatically call 
\family typewriter
marsadm leave-resource $res
\family default
 on some or all secondary nodes, such that the primary will be left alone
 and now has a chance to really delete its logfiles because no one else
 is any longer potentially needing it.
\end_layout

\begin_layout Enumerate
First-level kernelspace action, automatically executed when 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_4_gb
\end_layout

\end_inset


\family default
 + 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_3_gb
\end_layout

\end_inset


\family default
 + 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_2_gb
\end_layout

\end_inset


\family default
 + 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_1_gb
\end_layout

\end_inset


\family default
 is undershot:
\begin_inset Newline newline
\end_inset

a warning will be issued.
\end_layout

\begin_layout Enumerate
Second-level kernelspace action, automatically executed when 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_3_gb
\end_layout

\end_inset


\family default
 + 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_2_gb
\end_layout

\end_inset


\family default
 + 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_1_gb
\end_layout

\end_inset


\family default
 is undershot:
\begin_inset Newline newline
\end_inset

all locally secondary resources will delete local copies of transaction
 logfiles which are no longer needed locally.
 This is a desperate action of the kernel module.
\end_layout

\begin_layout Enumerate
Third-level kernelspace action, automatically executed when 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_2_gb
\end_layout

\end_inset


\family default
 + 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_1_gb
\end_layout

\end_inset


\family default
 is undershot:
\begin_inset Newline newline
\end_inset

all locally secondary resources will stop fetching transaction logfiles.
 This is a more desperate action of the kernel module.
 You don't want to get there (except for testing).
\end_layout

\begin_layout Enumerate
Last desperate kernelspace action when all else has failed and 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_free_space_1_gb
\end_layout

\end_inset


\family default
 is undershot:
\begin_inset Newline newline
\end_inset

all locally primary resources will enter 
\series bold
emergency mode
\series default
 (see description below in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Emergency-Mode"

\end_inset

).
 This is the most desperate action of the kernel module.
 You don't want to get there (except for testing).
\end_layout

\begin_layout Standard
In addition, the kernel module obeys a general global limit 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/required_total_space_0_gb
\end_layout

\end_inset

 +
\family default
 the sum of all of the above limits.
 When the 
\emph on
total size
\emph default
 of 
\family typewriter
/mars/
\family default
 undershots that sum, the kernel module refuses to start at all, because
 it assumes that it is senseless to try to operate MARS on a system with
 such low memory resources.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

The current level of emergency kernel actions may be viewed at any time
 via 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/mars_emergency_mode
\end_layout

\end_inset


\family default
.
\end_layout

\begin_layout Subsubsection
Throttling
\begin_inset CommandInset label
LatexCommand label
name "subsec:Throttling"

\end_inset


\end_layout

\begin_layout Standard
This not generally recommended.
 It may harm the IO performance from the viewpoint of your customers.
 Thus use it only as a 
\emph on
desperate
\emph default
 defense against overflow, by 
\series bold
throttling your performance pigs
\series default
.
\end_layout

\begin_layout Standard
Motivation: in rare cases, some users with 
\family typewriter
ssh
\family default
 access can do 
\emph on
very
\emph default
 silly things.
 For example,
\end_layout

\begin_layout Itemize
some users are creating their own backups via user-cron jobs, and they do
 it every 5 minutes.
 Some example guy created a zip archive (almost 1GB) by regularly copying
 his old zip archive into a new one, then appending deltas to the new one,
 and finally deleting the old archive.
 Every 5 minutes.
 Although almost never any new files were added to the archive.
 Essentially, he copied over his archive, for nothing.
 This led to massive bulk write requests, for ridiculous reasons.
\end_layout

\begin_layout Itemize
another user wrote his own shell script for his own private backup of his
 website, although there already is a daily system backup.
 He regularly made a complete copy of his entire webspace (more than 60GiB)
 via
\family typewriter
 cp -a
\family default
, then created a tarball out of the copy, uploaded it into the cloud, finally
 removed both the tarball and the complete filesystem copy.
 Each time, about 100GB was temporarily allocated (and replicated via MARS).
\end_layout

\begin_layout Standard
Typically, your hard disks / RAID systems allow much higher write IO rates
 than you can ever transport over a standard TCP network from your primary
 site to your secondary, at least over longer distances.
 Therefore, it is easy to create a such a high write load that it will be
 
\emph on
impossible
\emph default
 to replicate it over the network, 
\emph on
by construction
\emph default
.
\end_layout

\begin_layout Standard
MARS has some mechanism for throttling bulk writers whenever the network
 is weaker than your IO subsystem.
 It is off by default.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Notice that DRBD will 
\emph on
always
\emph default
 throttle your writes whenever the network forms a bottleneck, due to its
 synchronous operation mode.
 In contrast, MARS allows for buffering of performance peaks in the transaction
 logfiles.
 
\emph on
Only when
\emph default
 your buffer in 
\family typewriter
/mars/
\family default
 runs short (cf subsection 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Dimensioning-of-/mars/"

\end_inset

), MARS may be used for throttling your application writes.
\end_layout

\begin_layout Standard
There are a lot of screws named 
\family typewriter
/proc/sys/mars/write_throttle_*
\family default
 with the following meaning:
\end_layout

\begin_layout Description

\family typewriter
write_throttle_start_percent
\family default
 Whenever the used space in 
\family typewriter
/mars/
\family default
 is below this threshold, no throttling will occur at all.
 Only when this threshold is exceeded, throttling will start 
\emph on
slowly
\emph default
.
 Defaul value is 0, which means 
\begin_inset Quotes eld
\end_inset

off
\begin_inset Quotes erd
\end_inset

.
 Practical values for this coule be around 80%.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_end_percent
\family default
 Maximum throttling will occur once this space threshold is reached, i.e.
 the throttling is now at its maximum effect.
 A practical value is 90%, which is the default.
 When the actual space in 
\family typewriter
/mars/
\family default
 lies between 
\family typewriter
write_throttle_start_percent
\family default
 and 
\family typewriter
write_throttle_end_percent
\family default
, the strength of throttling will be interpolated linearly between the extremes.
 In practice, this should lead to an equilibrum between new input flow into
 
\family typewriter
/mars/
\family default
 and output flow over the network to secondaries.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_size_threshold_kb
\family default
 (readonly) This parameter shows the internal strength calculation of the
 throttling.
 Only write
\begin_inset Foot
status open

\begin_layout Plain Layout
Read requests are never throttled at all.
\end_layout

\end_inset

 requests exceeding this size (in KB) are throttled at all.
 Typically, this will hurt the bulk performance pigs first, while leaving
 ordinary users (issuing small requests) unaffected.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_ratelimit_kb
\family default
 Set the global IO rate in KB/s for those write requests which are throttled.
 In case of strongest
\begin_inset Foot
status open

\begin_layout Plain Layout
In case of lighter throttling, the input flow into 
\family typewriter
/mars/
\family default
 may be higher because small requests are not throttled.
\end_layout

\end_inset

 throttling, this parameters determines the input flow into 
\family typewriter
/mars/
\family default
.
 The default value is 10.000 KB/s.
 Please adjust this value to your application needs and to your environment.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_rate_kb
\family default
 (readonly) Shows the current rate of exactly those requests which are actually
 throttled (in contrast to 
\emph on
all
\emph default
 requests).
\end_layout

\begin_layout Description

\family typewriter
write_throttle_cumul_kb
\family default
 (logically readonly) Same as before, but the cumulative sum of all throttled
 requests since startup / reset.
 This value can be reset from userspace in order to prevent integer overflow.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_count_ops
\family default
 (logically readonly) Shows the cumulative number of throttled requests.
 This value can be reset from userspace in order to prevent integer overflow.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_maxdelay_ms
\family default
 Each request is delayed at most for this timespan.
 Smaller values will improve the responsiveness of your userspace application,
 but at the cost of potentially retarding the requests not sufficiently.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_minwindow_ms
\family default
 Set the minimum length of the measuring window.
 The measuring window is the timespan for which the average (throughput)
 rate is computed (see 
\family typewriter
write_throttle_rate_kb
\family default
).
 Lower values can increase the responsiveness of the controller algorithm,
 but at the cost of accuracy.
\end_layout

\begin_layout Description

\family typewriter
write_throttle_maxwindow_ms
\family default
 This parameter must be set sufficiently much greater than 
\family typewriter
write_throttle_minwindow_ms
\family default
.
 In case the flow of throttled operations pauses for some natural reason
 (e.g.
 switched off, low load, etc), this parameter determines when a completely
 new rate calculation should be started over
\begin_inset Foot
status open

\begin_layout Plain Layout
Motivation: if requests would pause for one hour, the measuring window could
 become also an hour.
 Of course, that would lead to completely meaningless results.
 Two requests in one hour is 
\begin_inset Quotes eld
\end_inset

incorrect
\begin_inset Quotes erd
\end_inset

 from a human point of view: we just have to ensure that averages are computed
 with respect to a reasonable maximum time window in the magnitude of 10s.
\end_layout

\end_inset

.
\end_layout

\begin_layout Section
Emergency Mode and its Resolution
\begin_inset CommandInset label
LatexCommand label
name "subsec:Emergency-Mode"

\end_inset


\end_layout

\begin_layout Standard
This section explains some implementation details.
 You may skip it.
\end_layout

\begin_layout Standard
When 
\family typewriter
/mars/
\family default
 is almost full and there is really absolutely no chance of getting rid
 of any local transaction logfile (or free some space in any other way),
 there is only one exit strategy: stop creating new logfile data.
\end_layout

\begin_layout Standard
This means that the ability for replication gets lost.
\end_layout

\begin_layout Standard
When entering emergency mode, the kernel module will execute the following
 steps for all resources where the affected host is acting as a primary:
\end_layout

\begin_layout Enumerate
Do a kind of 
\begin_inset Quotes eld
\end_inset

logrotate
\begin_inset Quotes erd
\end_inset

, but create a 
\emph on
hole
\emph default
 in the sequence of transaction logfile numbers.
 The 
\begin_inset Quotes eld
\end_inset

new
\begin_inset Quotes erd
\end_inset

 logfile is left empty, i.e.
 no data is written to it (for now).
 The hole in the numbering will prevent any secondaries from replaying any
 logfiles behind the hole (should they ever contain some data, e.g.
 because the emergency mode has been left again).
 This works because the secondaries are regularly checking the logfile numbers
 for contiguity, and they will refuse to replay anything which is not contiguous.
 As a result, the secondaries will be left in a consistent, but outdated
 state (at least if they already were consistent before that).
\end_layout

\begin_layout Enumerate
The kernel module writes back all data present in the temporary memory buffer
 (see figure in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-Transaction-Logger"

\end_inset

).
 This may lead to a (short) delay of user write requests until that has
 finished (typically fractions of a second or a few seconds).
 The reason is that the temporary memory buffer must not be increased in
 parallel during this phase (race conditions).
\end_layout

\begin_layout Enumerate
After the temporary memory buffer is empty, all local IO requests (whether
 reads or writes) are directly going to the underlying disk.
 This has the same effect as if MARS would not be present anymore.
 Transaction logging does no longer take place.
\end_layout

\begin_layout Enumerate
Any sync from any secondary is stopped ASAP.
 In case they are resuming their sync somewhen later, they will start over
 from the beginning (position 
\begin_inset Formula $0$
\end_inset

).
\end_layout

\begin_layout Standard
In order to leave emergency mode, the sysadmin should do the following steps:
\end_layout

\begin_layout Enumerate
Free enough space.
 For example, delete any foreign files on 
\family typewriter
/mars/
\family default
 which have nothing to do with MARS, or resize the 
\family typewriter
/mars/
\family default
 filesystem, or whatever.
\end_layout

\begin_layout Enumerate
The following control is intended for testing.
 If 
\family typewriter

\begin_inset Flex URL
status open

\begin_layout Plain Layout

/proc/sys/mars/mars_reset_emergency
\end_layout

\end_inset


\family default
 is off, now is the time to set it.
 By default, it should be already set.
\end_layout

\begin_layout Enumerate
Notice: as long as not enough space has been freed, a message containing
 
\family typewriter

\begin_inset Quotes eld
\end_inset

EMEGENCY MODE HYSTERESIS
\begin_inset Quotes erd
\end_inset


\family default
 (or similar) will be displayed by 
\family typewriter
marsadm view all
\family default
.
 As a consequence, any sync will be automatically halted.
 This applies to freshly invoked syncs also, for example created by 
\family typewriter
invalidate
\family default
 or 
\family typewriter
join-resource
\family default
.
\end_layout

\begin_layout Enumerate
On the secondaries, use 
\family typewriter
marsadm invalidate $res
\family default
 in order to request updating your outdated mirrors.
\end_layout

\begin_layout Enumerate
On the primary: 
\family typewriter
marsadm cron
\end_layout

\begin_layout Enumerate
As soon as emough space has been freed everywhere to leave the 
\family typewriter
EMEGENCY MODE HYSTERESIS
\family default
, sync should really start.
 Until that it had been halted.
\end_layout

\begin_layout Enumerate
Recommendation: check at secondaries that state 
\family typewriter
Orphan
\family default
 has been left after a while.
\end_layout

\begin_layout Standard
Alternatively, there is a more complicated method, which keeps more intermediate
 emergency backup replicas:
\end_layout

\begin_layout Enumerate
On 
\emph on
all
\emph default
 of your secondaries hostX:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm leave-resource mydata
\end_layout

\begin_layout Enumerate
At the primary hostA:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm cron
\end_layout

\begin_layout Enumerate
Wait until 
\family typewriter
df /mars
\family default
 shows no longer an overflow.
\end_layout

\begin_layout Enumerate
On the first secondary hostB:
\begin_inset Newline newline
\end_inset


\family typewriter
marsadm join-resource mydata /dev/lv/mydata
\end_layout

\begin_layout Enumerate
Wait until sync has finished at hostB.
\end_layout

\begin_layout Enumerate
If you have more than 2 replicas in total: proceed with step 4 at hostC,
 and so on.
 This time, you could join multipe resources in parallel, because you already
 have a life replica at hostB.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Expert advice, if you have only 2 replicas, and provided you have enough
 VG space: analogously to paragraph 
\begin_inset CommandInset ref
LatexCommand vref
reference "par:Retaining-a-Split"
plural "false"
caps "false"
noprefix "false"

\end_inset

 you may use 
\family typewriter
lvrename
\family default
 for keeping an outdated emergency backup before creating a new LV with
 the old name, and before re-joining the latter.
 Don't forget to remove your backup LV after sync has finished!
\end_layout

\begin_layout Chapter
Working with 
\family typewriter
marsadm
\family default
 commands
\family typewriter

\begin_inset CommandInset label
LatexCommand label
name "chap:The-Sysadmin-Interface"

\end_inset


\end_layout

\begin_layout Standard
This chapter is a kind of reference about the 
\family typewriter
marsadm
\family default
 tool.
 The sub-commands of 
\family typewriter
marsadm
\family default
 are grouped according to the topic they deal with.
\end_layout

\begin_layout Standard
Since MARS work 
\emph on
asynchronously
\emph default
 at metadata propagation level (which is 
\emph on
necessary
\emph default
 for long-distance replication over flaky networks), several commands are
 only 
\emph on
triggering
\emph default
 an action, but do not wait for its completion.
\end_layout

\begin_layout Standard
Such cases are indicated by the term 
\begin_inset Quotes eld
\end_inset

after a while
\begin_inset Quotes erd
\end_inset

.
 Please be aware that this 
\begin_inset Quotes eld
\end_inset

while
\begin_inset Quotes erd
\end_inset

 may last very long in case of network outages or bad firewall rules.
\end_layout

\begin_layout Standard
In the following tables, column 
\begin_inset Quotes eld
\end_inset

Cmp
\begin_inset Quotes erd
\end_inset

 means compatibility with DRBD.
 Please note that 100% exact compatibility is not possible, because of the
 asynchronous communication paradigm.
\end_layout

\begin_layout Standard
The following table documents common options which work with (almost) any
 
\family typewriter
marsadm
\family default
 command:
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="16" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Option
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--dry-run
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Run the command without actually creating symlinks or touching files or
 executing rsync.
 This option 
\emph on
should
\emph default
 be used first at any dangerous command, in order to check what would happen.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Don't use in scripts! Only use by hand!
\end_layout

\begin_layout Plain Layout

\size scriptsize
This option does not change the internal waiting logic for thois commands
 which emulate synchronous behaviour on top of the asynchronous communication
 paradigm.
 Many commands are waiting until the desired effect has succeeded.
 However, with 
\family typewriter
--dry-run
\family default
 the desired effect will never happen, so the command may wait forever (or
 abort with a timeout).
\end_layout

\begin_layout Plain Layout

\size scriptsize
In addition, this option can lead to additional aborts of the commands due
 to unmet conditions, which cannot be met because the symlinks are not actually
 created / altered.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Thus this option can give only a 
\series bold
rough estimate
\series default
 of what would happen later.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--force
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
almost
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Some preconditions are skipped, i.e.
 the command will / should work although some (more or less) vital preconditions
 are violated.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Instead of giving 
\family typewriter
--force
\family default
, you may alternatively prefix your command with 
\family typewriter
force-
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 THIS OPTION IS DANGEROUS!
\end_layout

\begin_layout Plain Layout

\size scriptsize
Use it only when you are absolutely sure that you know what you are doing!
\end_layout

\begin_layout Plain Layout

\size scriptsize
Use it only as a last resort if the same command without 
\family typewriter
--force
\family default
 has failed 
\emph on
for no good reason
\emph default
!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--parallel
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Only makes sense in combination with 
\family typewriter
all
\family default
.
 This is roughly equivalent to forking a bunch of parallel 
\family typewriter
marsdm
\family default
 processes, like in pseudo shell script notation: 
\family typewriter
for i in $resource_list; do marsadm $parameters $i & done; wait
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Several cluster managers are not re-entrant and may deadlock.
 First check whether this option is usable in your concrete environment!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--parallel=$number
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
--parallel
\family default
, but limit the parallelism degree to a maximum number of parallel processes.
 This may be useful for limiting the parallelism of startup processes, e.g.
 when kernel caches are cold, so the machine would get 
\emph on
overloaded
\emph default
 when too many resources would be starting in parallel.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Several cluster managers are not re-entrant and may deadlock.
 First check whether this option is usable in your concrete environment!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--ignore-sync
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Use this for a 
\emph on
planned
\emph default
 handover instead of 
\family typewriter
--force
\family default
.
 Only one precondition is relaxed: some sync may be running somewhere.
\end_layout

\begin_layout Plain Layout

\size scriptsize
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Careful when using this on extremely huge LVs where the sync may take serveral
 days, or weeks.
 It is your sysadmin decision what you want to prefer: restarting the sync,
 or planned handover.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--verbose
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Some (few) commands will become more speaky.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--timeout=$seconds
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Some commands require response from either the local kernel module, or from
 other cluster nodes.
 In order to prevent infinite waiting in case of network outages or other
 problems, the command will fail after the given timeout has been reached.
\end_layout

\begin_layout Plain Layout

\size scriptsize
When $seconds is -1, the command will wait forever.
\end_layout

\begin_layout Plain Layout

\size scriptsize
When $seconds is 0, the command will not wait in case any precondition is
 not met, und abort without performing an action..
\end_layout

\begin_layout Plain Layout

\size scriptsize
The default timeout is 5s.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--window=$seconds
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
The time window for checking the aliveness of other nodes in the network.
 When no symlink updates have been transferred from the other host since
 more than the window time, the host is considered dead.
 Default is 60s.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--keep-backups=$hours
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Only relevant for cron and link-purge-all.
 Old remains from dead / unreachable machines, and some backup data produced
 by join-cluster and split-cluster (potentially useful for experts), will
 be purged after this age.
 Default is 24 * 7 hours.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--threshold=$size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
The macros containing the substring 
\family typewriter
-threshold-
\family default
 or 
\family typewriter
-almost-
\family default
 are using this as a default value for approximation whether some data transfer
 (e.g.
 logfile and/or sync) has approximately completed.
 Default is 10MiB.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Notice: when data is continuously appended to the logfile, completeness
 may 
\emph on
never
\emph default
 be reached.
 Some data may always fly around somewhere in the network transfer channels.
\end_layout

\begin_layout Plain Layout

\size scriptsize
The $size argument may be a number optionally followed by one the lowercase
 characters k m g t p for indicating kilo mega giga tera or peta bytes as
 multiples of 1000.
 When using the corresponding uppercase character, multiples of 1024 are
 formed instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--host=$host
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
The command acts as if the command were executed on another host $host.
 This option should not be used regularly, because the local information
 in the symlink tree may be outdated or even wrong.
 Additionally, some local information like remote sizes of physical devices
 (e.g.
 remote disks) is not present in the symlink tree at all, or is wrong (reflectin
g only the 
\emph on
local
\emph default
 state).
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 THIS OPTION IS DANGEROUS!
\end_layout

\begin_layout Plain Layout

\size scriptsize
Use it only for final destruction of dead cluster nodes, see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Final-Destroy-of"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--ssh-port=$number
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
(deprecated) Only useful when the old 
\family typewriter
ssh
\family default
-based or
\family typewriter
 rsync
\family default
-based 
\family typewriter
{join,merge,split}-cluster
\family default
 or 
\family typewriter
join-resource
\family default
 commands are used.
 When newer
\family typewriter
 mars.ko
\family default
 and 
\family typewriter
marsadm
\family default
 versions are installed throughout the whole cluster, this is not needed
 anymore.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--no-ssh
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Avoid any potential timeouts / hangs caused by networks or firewalls, by
 explicitly disabling the old ssh-based communication method, and relying
 on the new MARS communication protocol (by defaut on port 7777).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--ip=$ip
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
hfill 
\end_layout

\end_inset

 or
\begin_inset Newline newline
\end_inset

--ip-$peer=$ip
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Override the IP information for the local host or 
\family typewriter
$peer
\family default
 at the command line.
 When this option is 
\emph on
not
\emph default
 given, the following rules apply in the following order:
\end_layout

\begin_layout Enumerate

\size scriptsize
lookup the IP for 
\family typewriter
$peer
\family default
 from the symlink tree in directory 
\family typewriter
/mars/ips/
\family default
.
\end_layout

\begin_layout Enumerate

\size scriptsize
so-called 
\emph on
probe data
\emph default
 from other hosts in the cluster.
 This tries to retrieve preliminary information as best as possible.
 It can however only work when the other peers are reachable, which also
 implies that in turn 
\emph on
their
\emph default
 currently configurred local peer IP must be correct.
\end_layout

\begin_layout Enumerate

\size scriptsize
Backups in 
\family typewriter
/mars/backup-$timestamp/
\family default
 as automatically created by several commands like 
\family typewriter
merge-cluster
\family default
 and 
\family typewriter
split-cluster
\family default
.
 When multiple historic backups are available, the 
\emph on
youngest
\emph default
 version will always win.
\end_layout

\begin_layout Enumerate

\size scriptsize
fallback to a DNS query via 
\family typewriter
/usr/bin/host
\family default
.
\end_layout

\begin_layout Enumerate

\size scriptsize
all local network interfaces are scanned by 
\family typewriter
/sbin/ip
\family default
 for IPv4 adresses, and the 
\emph on
first
\emph default
 one is taken.
 This may lead to wrong decisions if you have multiple network interfaces.
\end_layout

\begin_layout Plain Layout

\size scriptsize
In order to override this type of error-prone automatic IP detection and
 to explicitly tell the IP address of your 
\emph on
storage network
\emph default
 (which might be different from the ordinary IP address of your host), please
 use this option for maximum safety.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Usually you will need this only at 
\family typewriter
{create,join,merge}-cluster
\family default
 to determine any not yet known addresses.
 Typically, 
\family typewriter
{leave,split}-cluster
\family default
 are able to automatically detect historic information from the backups.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
--verbose
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Some (few) commands will become more speaky.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Cluster Operations
\begin_inset CommandInset label
LatexCommand label
name "sec:Cluster-Operations"

\end_inset


\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="10" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
create-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the 
\family typewriter
/mars/
\family default
 filesystem must be mounted and it must be empty (
\family typewriter
mkfs.ext4
\family default
, see instructions in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Setup-your-Cluster"

\end_inset

).
 The 
\family typewriter
mars.ko
\family default
 kernel module must 
\emph on
not
\emph default
 be loaded.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the initial symlink tree is created in 
\family typewriter
/mars/
\family default
.
 Additionally, the 
\family typewriter
/mars/uuid
\family default
 symlink is created for later distribution in the cluster.
 It uniquely indentifies the cluster in the world.
\end_layout

\begin_layout Plain Layout

\size scriptsize
This must be called exactly once at the initial primary.
\end_layout

\begin_layout Plain Layout
Hint: use the 
\family typewriter
--ip=
\family default
 option if you have multiple network interfaces.
\end_layout

\begin_layout Plain Layout
Full example on hostA: 
\family typewriter
marsadm --ip=192.168.2.101 create-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
join-cluster
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$host
\end_layout

\begin_layout Plain Layout
-or-
\end_layout

\begin_layout Plain Layout

\family typewriter
\size scriptsize
join-cluster
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$host
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$host_ip
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Preconditions: the cluster must have been already created with 
\family typewriter
create-cluster
\family default
 at another node 
\family typewriter
$host
\family default
 (optionally supplying an alternative IP address 
\family typewriter
$host_ip
\family default
).
 At your local node, the 
\family typewriter
/mars/
\family default
 filesystem must be mounted and it must be empty (
\family typewriter
mkfs.ext4
\family default
, see instructions in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Setup-your-Cluster"

\end_inset

).
 ,
\begin_inset space ~
\end_inset

The 
\family typewriter
mars.ko
\family default
 kernel module must be either loaded, or ssh must be working [exception:
 in old MARS versions before mars0.1astable101 the kernel module 
\emph on
must not
\emph default
 be loaded, and a working ssh connecttion to $host as root must work (without
 password), and
\family typewriter
rsync
\family default
 must be installed at all cluster nodes].
 In newer MARS versions >= mars0.1astable101, the old ssh-based method is
 automatically used as a fallback when the kernel module is forgotten to
 load.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the initial symlink tree 
\family typewriter
/mars/
\family default
 is replicated from the remote host 
\family typewriter
$host
\family default
, and the local host has been added as another cluster member.
\end_layout

\begin_layout Plain Layout

\size scriptsize
This must be called exactly once at every initial secondary node.
\end_layout

\begin_layout Plain Layout
Hint: use the 
\family typewriter
--ip=
\family default
 option if you have multiple interfaces on your local hostB.
\begin_inset Newline newline
\end_inset

Full example on hostB: 
\family typewriter
marsadm --ip=192.168.2.102 join-cluster hostA 192.168.2.101
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
leave-cluster
\begin_inset CommandInset label
LatexCommand label
name "leave-cluster"

\end_inset


\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the 
\family typewriter
/mars/
\family default
 filesystem must be mounted and it must contain a valid MARS symlink tree
 produced by the other 
\family typewriter
marsadm
\family default
 commands.
 The local node must no longer be member of any resource (see 
\family typewriter
marsadm leave-resource
\family default
).
 The kernel module should be loaded and the network should be operating
 in order to also propogate the effect to the other cluster nodes.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the local node is removed from the replicated symlink tree
 
\family typewriter
/mars/
\family default
 such that other nodes will cease to communicate with it after a while.
 The converse it not true: the local node may continue
\begin_inset Foot
status open

\begin_layout Plain Layout

\size scriptsize
Reason: 
\family typewriter
leave-cluster
\family default
 removes only its 
\emph on
own
\emph default
 IP address from 
\family typewriter
/mars/ips/
\family default
, but does not destroy the usual symmetry of the symlink tree by leaving
 the other IPs intact.
 Therefore, the local node will continue fetching updates from all nodes
 present in 
\family typewriter
/mars/ips/
\family default
.
 As an effect, the local node will 
\emph on
passively
\emph default
 mirror the symlinks of other cluster members, but not vice versa.
 There is no communication from the local node to the other ones, turning
 the local node into a 
\series bold
whitness
\series default
 according to some terminology from Distributed Systems.
 This is a feature, not a bug.
 It could be used for porst-mortem analysis, or for monitoring purposes.
 However, 
\emph on
deletions
\emph default
 of symlinks are not guaranteed to take place, so your whitness may 
\emph on
accumulate
\emph default
 thousands of old symlinks over a long time.
 If you want to eventually stop all communication to the local node, just
 run 
\family typewriter
rmmod
\family default
.
\end_layout

\end_inset

 passivley fetching the symlink tree.
 In order to really stop all communication, the kernel module should be
 unloaded afterwards (rmmod mars).
 The local 
\family typewriter
/mars/
\family default
 filesystem may be manually destroyed thereafte,r, e.g.
 for decommissioning of hardware.
 This is receommended for preventing 
\begin_inset Quotes eld
\end_inset

zombies
\begin_inset Quotes erd
\end_inset

 to resurrect by accidenct (human error).
\end_layout

\begin_layout Plain Layout

\size scriptsize
In case of an unintended hardware destruction (e.g.
 fire, water, ...) this command should be used on another healty cluster node
 $helper in order to finally remove $damaged from the cluster via the command
 
\family typewriter
marsadm leave-cluster --host=$damaged --force
\family default
.
 An example is explained in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "subsec:Final-Destroy-of"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Before leave-cluster, ensure that all other cluster nodes know that it is
 no longer participating in 
\emph on
any
\emph default
 resource!
\end_layout

\begin_layout Plain Layout

\size scriptsize
Hint: this can be usually achieved by marsadm leave-resource $resource –host=$da
maged –force
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
In case you cannot use 
\family typewriter
leave-cluster
\family default
 for any reason (e.g.
 complete network shutdown, no communication anymore possible at all), here
 is a last resort: destroy the 
\family typewriter
/mars/
\family default
 filesystem on the host 
\family typewriter
$deadhost
\family default
 you want to remove (e.g.
 by 
\family typewriter
mkfs
\family default
), or take other measures to 
\emph on
ensure
\emph default
 that it cannot be accidentally re-used in any way (e.g.
 physical destruction of the underlying RAID, 
\family typewriter
lvremove
\family default
, etc).
 On all other hosts, do 
\family typewriter
rmmod mars
\family default
, then delete the symlink 
\family typewriter
/mars/ips/ip-$deadhost
\family default
 everywhere by hand, and finally 
\family typewriter
modprobe mars
\family default
 again.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
Notice that the last 
\family typewriter
leave-resource
\family default
 operation does not delete the cluster as such.
 It just creates an 
\emph on
empty
\emph default
 cluster which has no longer any members.
 In particular, the cluster ID 
\family typewriter
/mars/uuid
\family default
 is 
\emph on
not
\emph default
 removed, deliberately
\begin_inset Foot
status open

\begin_layout Plain Layout

\size scriptsize
This is a feature, not a bug.
 The 
\family typewriter
uuid
\family default
 is created once, but alterered anywhere.
 An exception is 
\family typewriter
marsadm merge-cluster
\family default
 (see there).
 The only way to get rid of the uuid is 
\emph on
external
\emph default
 deletion (not by 
\family typewriter
marsadm
\family default
) 
\emph on
together(!)
\emph default
 with all other contents of 
\family typewriter
/mars/
\family default
.
 This prevents you from accidentally merging half-dead remains which could
 have survived a disaster for any reason, such as snapshotting filesystems
 / VMs or whatever.
\end_layout

\end_inset

.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Before you can re-use 
\emph on
any
\emph default
 left-over 
\family typewriter
/mars/
\family default
 filesystem for creating / joining a new / different cluster, you 
\emph on
must
\emph default
 create a fresh filesystem (see instructions in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Setup-your-Cluster"

\end_inset

) via 
\family typewriter
mkfs.ext4
\family default
.
 Exception: 
\family typewriter
marsadm merge-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
merge-cluster
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$host
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Preconditions: The set of resources at the local cluster (transitively)
 and at the other cluster as addressed by some foreign member 
\family typewriter
$host
\family default
 (transitively) must be disjoint.
 
\emph on
All(!)
\emph default
 hosts must be mutually reachable via the MARS ports (default 7777 to 7779).
 Since 
\family typewriter
mars0.1astable114,
\family default
 
\family typewriter
ssh
\family default
 and 
\family typewriter
rsync
\family default
 are no longer required, provided that 
\emph on
both
\emph default
 clusters have been fully updated.
 Otherwise ssh must be working between all members of both clusters, without
 password (e.g.
 via ssh-agent).
\end_layout

\begin_layout Plain Layout

\size scriptsize
Create the union of both clusters, consisting of the union of all participating
 machines (transitively).
 Resource memberships are unaffected.
 This is useful for creating a 
\begin_inset Quotes eld
\end_inset

virtual LVM cluster
\begin_inset Quotes erd
\end_inset

 where resources can be migrated later via 
\family typewriter
join-resource
\family default
 / 
\family typewriter
leave-resource
\family default
 operations.
 Usage examples can be found in the Football sub-project of MARS.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
Attention! use a newer version of MARS.
 The old branch 
\family typewriter
0.1.y
\family default
 does not scale well in number of cluter members, because it evolved from
 a lab prototype with 
\begin_inset Formula $O(n^{2})$
\end_inset

 behaviour at metadata exchange.
 Never exceed the maximum cluster members as described in appendix 
\begin_inset CommandInset ref
LatexCommand vref
reference "chap:Technical-Data-MARS"

\end_inset

.
 For safety, you should better stay at 1/2 of the numbers mentioned there.
 Use 
\family typewriter
split-cluster
\family default
 for going back to smaller clusters again after your background data migration
 has completed.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Future versions of MARS will be constructed for very big clusters in the
 range of thousands of nodes.
 Development has not yet stabilized there, and operational experiences are
 missing at the moment.
 Be careful until official announcements are appearing in the ChangeLog,
 reporting of operational experiences from the 1&1 big cluster at metadata
 level.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
merge-cluster-check
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$host
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Check in advance whether the set of resources at the local cluster and at
 the other cluster 
\family typewriter
$host
\family default
 are disjoint.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
split-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
This is almost the inverse operation of 
\family typewriter
merge-cluster
\family default
: it determines the minimum sub-cluster groups participating in some common
 resources.
 Then it splits the cluster memberships such that unnecessary connections
 between non-related nodes are interrupted.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Use this for avoidance of too big clusters.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Since 
\family typewriter
mars0.1astable114,
\family default
 
\family typewriter
ssh
\family default
 and 
\family typewriter
rsync
\family default
 are no longer required, provided that 
\emph on
all
\emph default
 hosts are mutually reachable over the default metadata communication port
 7777.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Waiting"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
update-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Waiting"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
create-uuid
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
Deprecated.
 Only for compatibility with very old version light0.1beta05 or earlier.
 Will disappear somewhen in future.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Precondition: the 
\family typewriter
/mars/
\family default
 filesystem must be mounted.
 A 
\family typewriter
uuid
\family default
 (such as automatically created by recent versions of 
\family typewriter
marsadm create-cluster
\family default
) must not already exist; i.e.
 you have a very old and outdated symlink tree.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the 
\family typewriter
/mars/uuid
\family default
 symlink is created for later distribution in the cluster.
 It uniquely indentifies the cluster in the world.
\end_layout

\begin_layout Plain Layout

\size scriptsize
This must be called at most once at the current primary.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Resource Operations
\begin_inset CommandInset label
LatexCommand label
name "sec:Resource-Operations"

\end_inset


\end_layout

\begin_layout Standard
Common precondition for all resource operations is that the 
\family typewriter
/mars/
\family default
 filesystem is mounted, that it contains a valid MARS symlink tree produced
 by other 
\family typewriter
marsadm
\family default
 commands (including a unique 
\family typewriter
uuid
\family default
), that your current node is a valid member of the cluster, and that the
 kernel module 
\family typewriter
mars.ko
\family default
 is loaded.
 When communication is impossible due to network outages or bad firewall
 rules, most commands will succeed, but other cluster nodes may take a long
 time to notice your changes.
\end_layout

\begin_layout Standard
Instead of executing 
\family typewriter
marsadm
\family default
 commands serveral times for each resource argument, you may give the special
 resource argument 
\family typewriter
all
\family default
.
 This work even when combined with 
\family typewriter
--force
\family default
, but be cautious when giving dangerous command combinations like 
\family typewriter
marsadm delete-resource --force all
\family default
.
\end_layout

\begin_layout Standard
In newer versions of 
\family typewriter
marsadm
\family default
, you may give a comma-separated list of resource names in place of 
\family typewriter
all
\family default
.
 This way, you have more fine-grained control over the set of resource names
 you want to use.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Beware when combining this with 
\family typewriter
--host=somebody
\family default
.
 In some very rare cases, like final destruction of a whole datacenter after
 an earthquake, you might need a combination like 
\family typewriter
marsadm --host=defective delete-resource --force all
\family default
.
 Don't use such combinations if you don't need them 
\emph on
really
\emph default
! You can easily shoot yourself in your head if you are not carefully operating
 such commands!
\end_layout

\begin_layout Subsection
Resource Creation / Deletion / Modification
\begin_inset CommandInset label
LatexCommand label
name "subsec:Resource-Creation"

\end_inset


\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="8" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
create-resource
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$disk_dev
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

[$mars_name]
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

[$size]
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the resource argument 
\family typewriter
$res
\family default
 must denote a 
\emph on
new
\emph default
 (not yet existing) resource name in the cluster.
 The argument 
\family typewriter
$disk_dev
\family default
 must denote an absolute path to a usable local block device, its size must
 be greater zero.
 When the optional 
\family typewriter
$mars_name
\family default
 is given, that name must not already exist on the local node; when not
 given, 
\family typewriter
$mars_name
\family default
 defaults to 
\family typewriter
$res
\family default
.
 When the optional 
\family typewriter
$size
\family default
 argument is given, it must be a number, optionally followed by a lowercase
 suffix 
\family typewriter
k
\family default
, 
\family typewriter
m
\family default
, 
\family typewriter
g
\family default
, 
\family typewriter
t
\family default
, or 
\family typewriter
p
\family default
 (denoting size factors as multiples of 1000), or an uppercase suffix 
\family typewriter
K
\family default
, 
\family typewriter
M
\family default
, 
\family typewriter
G
\family default
, 
\family typewriter
T
\family default
 or 
\family typewriter
P
\family default
 (denoting size factors as multiples of 1024).
 The given size must not exceed the actual size of 
\family typewriter
$disk_dev
\family default
.
 It will specify the future resource size as shown by 
\family typewriter
marsadm view-resource-size $res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the resource 
\family typewriter
$res
\family default
 is created, the inital role of the current node is primary.
 The corresponding symlink tree information is asynchonously distributed
 in the cluster (in the background).
 The device 
\family typewriter
/dev/mars/$mars_name
\family default
 should appear after a while.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Notice: when 
\family typewriter
$size
\family default
 is strictly smaller than the size of 
\family typewriter
$disk_dev
\family default
, you will unnecessarily waste some space..
\end_layout

\begin_layout Plain Layout

\size scriptsize
This must be called exactly once for any new resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
join-resource
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$disk_dev
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

[$mars_name]
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the resource argument 
\family typewriter
$res
\family default
 must denote an already existing resource in the whole cluster (i.e.
 its symlink tree information must have been received; use 
\family typewriter
marsadm wait-cluster
\family default
 for achieving this).
 The resource must have a designated primary, and it must no be in emergency
 mode.
 There must not exist a split brain in the cluster.
 The local node must not be already member of that resource.
 The argument 
\family typewriter
$disk_dev
\family default
 must denote an absolute path to a usable (but currently unused) local block
 device, its size must be greater or equal to the logical size of the resource.
 When the optional 
\family typewriter
$mars_name
\family default
 is given, that name must not already exist on the local node; when not
 given, 
\family typewriter
$mars_name
\family default
 defaults to 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the current node becomes a member of resource 
\family typewriter
$res
\family default
, the inital role is secondary.
 The initial full sync should start after a while.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Notice: when the size of $disk_dev is strictly greater than the size of
 the resource, you will unnecessarily waste some space.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\size scriptsize
After a while, state 
\family typewriter
Orphan
\family default
 should be left.
 Don't forget to regularly monitor for longer occurrences of 
\family typewriter
Orphan
\family default
!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
leave-resource
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node must be a member of the resource 
\family typewriter
$res
\family default
; its current role must be secondary.
 It must be detached (see 
\family typewriter
marsadm down
\family default
).
 The kernel module should be loaded and the network should be operating
 in order to also propogate the effect to the other cluster nodes.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the local node is no longer a member of 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Notice: as a side effect for other nodes, their 
\family typewriter
log-delete
\family default
 may now become possible, since the current node does no longer count as
 a candidate for logfile application.
 As another side effect, split brain situation may be (partly) resolved
 by this.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Please notice that this command 
\emph on
may likely 
\emph default
resolve split brain (but cannot guarantee in general).
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
The contents of the disk is not changed by this command.
 Before issuing this command, check whether the disk appears to be locally
 consistent (see 
\family typewriter
view-is-consistent
\family default
)! After giving this command, any internal information indicating the consistenc
y state will be gone, and you will no longer be able to guess consistency
 properties.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 When you are 
\emph on
sure
\emph default
.that the disk was consistent before (or is now by manually checking it),
 you may re-create a new resource out of it via 
\family typewriter
create-resource
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
In case of an eventual node loss (e.g.
 fire, water, ...) this command needs to be used on another node $helper in
 order to finally remove all the resources $damaged from the cluster via
 the command 
\family typewriter
marsadm leave-resource $res --host=$damaged --force
\family default
.
 Details are in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "subsec:Final-Destroy-of"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
delete-resource
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the resource must be empty (i.e.
 all cluster members must have left via 
\family typewriter
leave-resource
\family default
).
 This precondition is overridable by 
\family typewriter
--force
\family default
, increasing the danger to maximum! It is even possible to combine 
\family typewriter
--force
\family default
 with an invalid resource argument and an invalid 
\family typewriter
--host=somebodyelse
\family default
 argument in order to desperately try to destroy remains of incomplete or
 pysically damaged hardware.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: all cluster members will somewhen be forcefully removed from
 
\family typewriter
$res
\family default
.
 In case of network interruptions, the forced removal may take place far
 in the future.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 THIS COMMAND IS 
\emph on
VERY
\emph default
 DANGEROUS!
\end_layout

\begin_layout Plain Layout

\size scriptsize
Use this only in desperate situations, and only manually.
 Don't call this from scripts.
 You are forcefully using a sledgehammer, even without 
\family typewriter
--force
\family default
! The danger is that the 
\emph on
true
\emph default
 state of other cluster nodes cannot be known in general, e.g.
 network problems etc.
 Even when it were known, it could be compromised by 
\series bold
byzantine failures
\series default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
It is strongly advised to try this command with 
\family typewriter
--dry-run
\family default
 first.
\end_layout

\begin_layout Plain Layout

\size scriptsize
When combined with 
\family typewriter
--force
\family default
, this command will definitely 
\series bold
murder
\series default
 other cluster nodes, possibly after a long while, and even when they are
 operating in primary mode / having split brains / etc.
 However, there is no guarantee that other cluster nodes will be 
\emph on
really
\emph default
 dead – it is (theoretically) possible that they remain only 
\emph on
half
\emph default
 
\emph on
dead
\emph default
.
 For example, a half dead node may continue to write data to 
\family typewriter
/mars/
\family default
 and thus lead to overflow somewhen.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

This command implies a forceful detach, possibly destroying consistency.
\end_layout

\begin_layout Plain Layout

\size scriptsize
It is similar in spirit to 
\series bold
STONITH
\series default
, but on cluster level, affection all known resource members.
 In particular, when a cluster node was operating in primary mode (
\family typewriter
/dev/mars/mydata
\family default
 being continuously in use), the forceful detach cannot be carried out until
 the device is completely unused.
 In the meantime, the current transaction logfile will be appended to, but
 the file 
\emph on
might
\emph default
 be already unlinked (orphan file filling up the disk).
 After the forceful detach, the underlying disk need not be consistent (although
 MARS does its best).
 Since this command deletes any symlinks which normally would indicate the
 consistency state, no guarantees about consistency can be given after this
 
\emph on
in general
\emph default
! Always check consistency by hand!
\end_layout

\begin_layout Plain Layout

\size scriptsize
When possible / as soon as possible, check the local state on the other
 nodes in order to 
\emph on
really
\emph default
 shutdown the resource everywhere (e.g.
 to 
\emph on
really
\emph default
 unuse the 
\family typewriter
/dev/mars/mydata
\family default
 device, etc).
\end_layout

\begin_layout Plain Layout

\size scriptsize
After this command, you 
\emph on
should
\emph default
 rebuild the resource under a different name, in order to avoid any clashes
 caused by unexpected resurrection of 
\begin_inset Quotes eld
\end_inset

dead
\begin_inset Quotes erd
\end_inset

 or 
\begin_inset Quotes eld
\end_inset

half-dead
\begin_inset Quotes erd
\end_inset

 zombie nodes (beware of shapshot / restores on virtual machines!!).
 MARS does its best to avoid problems even in case the new resource name
 should equal the old one, but there can be 
\emph on
no guarantee
\emph default
 in all possible failure scenarios / usage scenarios.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Whenever possible, prefer 
\family typewriter
leave-resource
\family default
 over this kind of sledgehammer!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
activate-guest
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the current host must be a cluster member (see commands 
\family typewriter
join-cluster
\family default
 and 
\family typewriter
merge-cluster
\family default
), but need not (yet) be a resource member.
 The resource must exist somewhere else in the cluster.
 No additional storage is needed, except a few kilo or megabytes (typically)
 for the symlink tree.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: symlink updates with other resource members and/or guests
 are more frequently.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Consequently, marsadm commands and macros with the 
\family typewriter
--host=
\family default
 option may be used for remote state inspection, etc.
\end_layout

\begin_layout Plain Layout

\family typewriter
\size scriptsize
marsadm view all
\family default
 will display the guest and its status.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
deactivate-guest
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the resource must exist.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any previous pure 
\family typewriter
activate-guest
\family default
 is rolled back.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
After about a month,, 
\family typewriter
marsadm cron
\family default
 will also remove the guest relationship.
 This is to protect you from long-term accumulation of unnecessary guest
 relationships, which are intended only for 
\emph on
temporary
\emph default
 purposes (in constrast to 
\emph on
full
\emph default
 resource memberships requiring storage space for keeping 
\emph on
persistent
\emph default
 replicas).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-resource
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

{is-,}{attach,
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 primary,
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 device}{-off,}
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Waiting"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Operation of the Resource
\begin_inset CommandInset label
LatexCommand label
name "subsec:Operation-of-the"

\end_inset


\end_layout

\begin_layout Standard
Common preconditions are the preconditions from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Resource-Operations"

\end_inset

, plus the respective resource 
\family typewriter
$res
\family default
 must exist, and the local node must be a member of it.
 With the single exception of 
\family typewriter
attach
\family default
 itself, all other operations must be started in 
\family typewriter
attached
\family default
 state.
\end_layout

\begin_layout Standard
When 
\family typewriter
$res
\family default
 has the special reserved value 
\family typewriter
all
\family default
, the following operations will work on all resources where the current
 node is a member (analogously to DRBD).
\end_layout

\begin_layout Standard
With newer versions of 
\family typewriter
marsadm
\family default
, you can also give a list of comma-separated resource names in place of
 
\family typewriter
all
\family default
.
\end_layout

\begin_layout Standard
\noindent

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="44" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
attach
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
yes
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local disk belonging to $res is not in use by anyone else.
 Its contents has not been altered in the meantime since the last 
\family typewriter
detach
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Mounting 
\emph on
read-only
\emph default
 is allowed during the detached phase.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
However, be careful! If you 
\emph on
accidentally
\emph default
 forget to give the right readonly-mount flags, if you use 
\family typewriter
fsck
\family default
 in repair mode inbetween, or alter the disk content in any other way (beware
 of LVM snapshots / restores etc), you will almost certainly produce an
 
\series bold
unnoticed inconsistency
\series default
 (not reported by 
\family typewriter
view-is-consistent
\family default
)! MARS has 
\emph on
no chance
\emph default
 to notice suchalike!
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: MARS uses the local disk and is able to work with it (e.g.
 replay logfiles on it).
\end_layout

\begin_layout Plain Layout

\size scriptsize
Note: the local disk is opened in exclusive read-write mode.
 This should protect against most common misuse, such as opening the disk
 in parallel to MARS.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
However, this does not necessarily protect against non-exclusive openers.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
detach
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
yes
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local 
\family typewriter
/dev/mars/mydata
\family default
 device (when present) is no longer opened by anybody.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the local disk belonging to $res is no longer in use.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
In contrast to DRBD, you need not explicitly pause syncing, fetching, or
 replaying 
\emph on
to
\emph default
 (as apposed to 
\emph on
from
\emph default
) the local disk.
 These processes are automatically paused.
 Another difference to DRBD: the fetch / replay processes etc will usually
 
\emph on
automatically
\emph default
 resume after re-attach, as far as possible in the respective new situation.
 This will usually work even over 
\family typewriter
rmmod
\family default
 or reboot cycles, since the internal symlink tree will automatically persist
 all todo switches for you (c.f.
 section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-State-of"

\end_inset

).
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
Notice: only 
\emph on
local
\emph default
 transfer operations 
\emph on
to
\emph default
 the local disk are paused by a detach.
 When another node is remotely running a sync 
\emph on
from
\emph default
 your local disk, it will likely remain in use for remote reading.
 The reason is that the server part of MARS is operating purely passively,
 in order serve all remote requests as best as possible (similar to the
 original Unix philosophy).
 In order to really stop all accesses, do a 
\family typewriter
pause-sync
\family default
 on all other resource member where a sync is currently running.
 You may also try 
\family typewriter
pause-sync-global
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
WARNING! After this, and ather having paused any remote data access, you
 might use the underlying disk for your own purposes, such as test-mounting
 it in 
\emph on
readonly
\emph default
 mode.
 
\series bold
Don't modifiy
\series default
 its contents in any way! Not even by an 
\family typewriter
fsck
\family default

\begin_inset Foot
status open

\begin_layout Plain Layout

\size scriptsize
Some (but not all) 
\family typewriter
fsck
\family default
 tools for some filesystems have options to start only a test repair / verify
 mode / dry run, without doing actual modifications to the data.
 Of course, these modes 
\emph on
can
\emph default
 be used.
 But be really sure! Double-check for the right options!
\end_layout

\end_inset

! Otherwise, you will have inconsistencies 
\emph on
guaranteed
\emph default
.
 MARS has no way for knowing of any modifications to your disk when bypassing
 
\family typewriter
/dev/mars/*
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
In case you accidentally modified the underlying disk at the 
\emph on
primary
\emph default
 side, you may choose to resolve the inconsistencies by 
\family typewriter
marsadm invalide $res
\family default
 on 
\emph on
each
\emph default
 secondary.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-sync
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-sync-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-sync-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any sync operation targeting the local disk (when not yet
 completed) is paused after a while (cf section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-State-of"

\end_inset

).
 When successfully completed, this operation will remember the switch state
 forever and automatically become relevant if a sync is needed again (e.g.
 
\family typewriter
invalidate
\family default
 or 
\family typewriter
resize
\family default
).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-sync-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
*-local
\family default
, but operates on all members of the resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-sync
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
resume-sync-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-sync-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: additionally, a primary must be designated, and it must not
 be in emergency mode.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any sync operation targeting the local disk (when not yet
 completed) is resumed after a while.
 When completed, this operation will remember the switch state forever and
 become relevant if a sync is needed again (e.g.
 
\family typewriter
invalidate
\family default
 or 
\family typewriter
resize
\family default
).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-sync-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
*-local
\family default
, but operates on all members of the resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-fetch
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-fetch-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-fetch-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
 The resource 
\emph on
should
\emph default
 be in secondary role.
 Otherwise the switch has 
\emph on
no
\emph default
 
\emph on
immediate
\emph default
 effect, but will come (possibly unexpectedly) into effect whenever secondary
 role is entered later for whatever reason.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any transfer of (parts of) transaction logfiles which are
 present at another primary host to the local 
\family typewriter
/mars/
\family default
 storage are paused at their current stage.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
This switch works independently from 
\family typewriter
{pause,resume}-replay
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-fetch-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
*-local
\family default
, but operates on all members of the resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-fetch
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
resume-fetch-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-fetch-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
 The resource 
\emph on
should
\emph default
 be in secondary role.
 Otherwise the switch has 
\emph on
no
\emph default
 
\emph on
immediate
\emph default
 effect, but will come (possibly unexpectedly) into effect whenever secondary
 role is entered later for whatever reason.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any (parts of) transaction logfiles which are present at
 another primary host shouldl be transferred to the local 
\family typewriter
/mars/
\family default
 storage as far as not yet locally present.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
This works independently from 
\family typewriter
{pause,resume}-replay
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-fetch-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
*-local
\family default
, but operates on all members of the resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-replay
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-replay-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-replay-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
 The resource 
\emph on
should
\emph default
 be in secondary role.
 Otherwise the switch has 
\emph on
no
\emph default
 
\emph on
immediate
\emph default
 effect, but will come (possibly unexpectedly) into effect whenever secondary
 role is entered later for whatever reason.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any local replay operations of transaction logfiles to the
 local disk are paused at their current stage.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
This works independently from 
\family typewriter
{pause,resume}-fetch
\family default
 resp.
 
\family typewriter
{dis,}connect
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
pause-replay-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
*-local
\family default
, but operates on all members of the resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-replay
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-replay-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-replay-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: must be in secondary role.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: any (parts of) locally existing transaction logfiles (whether
 replicated from other hosts or produced locally) are started for replay
 to the local disk, as far as they have not yet been applied.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resume-replay-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Like 
\family typewriter
*-local
\family default
, but operates on all members of the resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
connect
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
connect-local
\family default
 and to 
\family typewriter
resume-fetch-local
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Note: although this sounds similar to DRBD's 
\family typewriter
drbdadm connect
\family default
, there are subtle differences.
 DRBD has exactly one connection per resource, which is associated with
 
\emph on
pairs
\emph default
 of nodes.
 In contrast, MARS may create multiple connections per resource at runtime,
 and these are associated with the 
\emph on
target
\emph default
 host (not with 
\emph on
pairs
\emph default
 of hosts).
 As a consequence, the fetch may 
\emph on
potentially
\emph default
 occur from any other source host which happens to be reachable (although
 the current implementation prefers the current designated primary, but
 this may change in future).
 In addition, 
\family typewriter
marsadm disconnect
\family default
 does not stop 
\emph on
all
\emph default
 communication.
 It only stops fetching logfiles.
 The symlink updates running in background (default port 7777) are 
\emph on
not
\emph default
 stopped, in order to always propagate as much metadata as possible throughout
 the cluster.
 In case of a later incident, chances will be higher for a better knowledge
 of the 
\emph on
real
\emph default
 state of the cluster.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
connect-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
resume-fetch-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
connect-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
resume-fetch-global
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
disconnect
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
disconnect-local
\family default
 and to 
\family typewriter
pause-fetch-local
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
See above note at 
\family typewriter
connect
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
disconnect-local
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-fetch-local
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
disconnect-global
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
partly
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-fetch-global
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
up
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
attach
\family default
 followed by 
\family typewriter
resume-fetch
\family default
 followed by 
\family typewriter
resume-replay
\family default
 followed by 
\family typewriter
resume-sync
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
down
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
pause-sync
\family default
 followed by 
\family typewriter
pause-fetch
\family default
 followed by 
\family typewriter
pause-replay
\family default
 followed by 
\family typewriter
detach
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
Hint: consider to prefer plain 
\family typewriter
detach
\family default
 over this, because 
\family typewriter
detach
\family default
 will remember the last state of all switches, while 
\family typewriter
down
\family default
 will 
\emph on
not
\emph default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
primary
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
almost
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
There are three variants:
\end_layout

\begin_layout Plain Layout
Variant 1: planned handover (no 
\family typewriter
--force
\family default
)
\end_layout

\begin_layout Plain Layout

\size scriptsize
Precondition: sync must have finished at any resource member.
 All relevant transaction logfiles must be either already locally present,
 or be fetchable (see 
\family typewriter
marsadm up
\family default
, or low-level commands 
\family typewriter
resume-fetch
\family default
 and 
\family typewriter
resume-replay
\family default
).
 When some logfile data is locally missing, there must be enough space on
 
\family typewriter
/mars/
\family default
 to fetch it.
 Any replay must not have been interrupted by a replay error (see macro
 %replay-code{} or diskstate 
\family typewriter
DefectiveLog
\family default
).
 The current designated primary must be reachable over network.
 When there is no designated primary (i.e.
 
\family typewriter
marsadm secondary
\family default
 had been executed before, which is explicitly 
\emph on
not recommended
\emph default
), at least the old primary must be reachable.
 The (old) primarie's virutal device 
\family typewriter
/dev/mars/mydata
\family default
 must not be in use any more (see 
\family typewriter
marsadm wait-umount
\family default
).
 A split brain must not already exist.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the current host is in primary role; 
\family typewriter
/dev/mars/$dev_name
\family default
 appears locally and is usable; the equivalent of 
\family typewriter
marsadm up $res
\family default
 has been executed as a safeguard against any forgotten previous 
\family typewriter
pause-*
\family default
 operations.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Switches the 
\series bold
designated primary
\series default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Description of the 
\series bold
Handover
\series default
 protocol (when 
\family typewriter
--force
\family default
 is not given): when another host is currently primary, it is first asked
 to leave its primary role.
 When systemd templates are active, this will be automatically triggered
 via 
\family typewriter
systemctl stop $stop_unit
\family default
.
 Otherwise, you are resposible for stopping the load yourself, and you dhoulf
 use 
\family typewriter
marsadm wait-umount
\family default
 in advance for checking.
 Anyway, the handover procol s waiting until the former primary has actually
 become secondary.
 After that, the local host is requested to become primary.
 Before actually becoming primary, all relevant logfiles are transferred
 over the network and replayed, in order to avoid accidental creation of
 split brain as best as possible
\begin_inset Foot
status open

\begin_layout Plain Layout

\size scriptsize
Note that split brain avoidance is 
\series bold
best effort
\series default
 and cannot be guaranteed in general.
 For example, it may be impossible to avoid split brain in case of long-lasting
 network outages.
\end_layout

\end_inset

.
 Only after that, 
\family typewriter
/dev/mars/$dev_name
\family default
 will appear.
 When network transfers of the symlink tree are very slow (or currently
 impossible), this command may take a very long time.
\end_layout

\begin_layout Plain Layout

\size scriptsize
In case a split brain is already detected at the initial situation, the
 local host will refuse to switch the designated primary without 
\family typewriter
--force
\family default
.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 In case of 
\begin_inset Formula $k>2$
\end_inset

 replicas: if you want to handover between host 
\family typewriter
A
\family default
 and 
\family typewriter
B
\family default
 while a sync is currently running at host 
\family typewriter
C
\family default
, you have the following options:
\end_layout

\begin_layout Enumerate

\size scriptsize
wait until the sync has finished (see macro 
\family typewriter
sync-rest
\family default
, or 
\family typewriter
marsadm view
\family default
 in general).
\end_layout

\begin_layout Enumerate

\size scriptsize
do a 
\family typewriter
leave-resouce
\family default
 on host 
\family typewriter
C
\family default
, and later 
\family typewriter
join-resource
\family default
 after the handover completed successfully.
\end_layout

\begin_layout Enumerate

\size scriptsize
use the option 
\family typewriter
--ignore-sync
\family default
, which leads to a restart of the running sync from position 0.
\end_layout

\begin_layout Plain Layout
Variant 2: planned handover (no 
\family typewriter
--force
\family default
) with sync abort (
\family typewriter
--ignore-sync
\family default
)
\end_layout

\begin_layout Plain Layout

\size scriptsize
2) 
\series bold
Handover ignoring running syncs,
\series default
 by adding the option 
\family typewriter
--ignore-sync
\family default
.
 Any running syncs will restart from scratch, in order to ensure consistency.
 Use this only when the planned handover is more important than the sync
 time.
\end_layout

\begin_layout Plain Layout
Variant 3: unplanned failover (
\family typewriter
--force
\family default
)
\end_layout

\begin_layout Plain Layout

\size scriptsize
3) 
\series bold
Forced switching
\series default
: by giving –force while 
\family typewriter
pause-fetch
\family default
 is active (but not 
\family typewriter
pause-replay
\family default
), most preconditions are ignored, and MARS does its best to actually become
 primary even if some logfiles are missing or incomplete or even defective.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\family typewriter
\size scriptsize
primary --force
\family default
 is a potentially harmful variant, because it will provoke a split brain
 in most cases, and therefore in turn will lead to 
\series bold
data loss
\series default
 because one of your split brain versions must be discarded later in order
 to resolve the split brain (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Split-Brain-Resolution"

\end_inset

).
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\series bold
\size scriptsize
Never
\series default
 call 
\family typewriter
primary --force
\family default
 when planned handover via 
\family typewriter
primary
\family default
 without 
\family typewriter
--force
\family default
 is sufficient! If 
\family typewriter
primary
\family default
 without 
\family typewriter
--force
\family default
 complains that the device is in use at the former primary side, take it
 seriously! Don't override with 
\family typewriter
--force
\family default
, but rather umount
\begin_inset Foot
status open

\begin_layout Plain Layout

\size scriptsize
A common misconception is when people think that they can keep their filesystem
 mounted without provoking a split brain, because they have their application
 stopped and thus don't write any data into the filesystem.
 This is a wrong idea, because filesystems may write some metadata, like
 booking information, even after hours or days of inactivity.
 Therefore MARS insists that the device is no longer in use before any handover
 can take place.
\end_layout

\end_inset

 the device at the other side!
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Only use 
\family typewriter
primary --force
\family default
 when something is 
\emph on
already broken
\emph default
, such as a network outage, or a node crash, etc.
 During ordinary operations (network OK, nodes OK), you should never need
 
\family typewriter
primary --force
\family default
!
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 If you umount 
\family typewriter
/dev/mars/mydata
\family default
 on the old primary 
\family typewriter
A
\family default
, and then wait until 
\family typewriter
marsadm view
\family default
 (or another suitable macro) on the target host 
\family typewriter
B
\family default
 shows that everything is 
\family typewriter
UpToDate
\family default
, you have some 
\emph on
chance
\emph default
 for avoiding a split brain even with 
\family typewriter
primary --force
\family default
.
 However, there is no guarantee.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset


\family typewriter
\size scriptsize
 primary --force
\family default
 switches the 
\emph on
designated
\emph default
 primary.
 In some extremely rare cases, when 
\emph on
multiple
\emph default
 faults have accumulated in a 
\emph on
weird
\emph default
 situation, it 
\emph on
might
\emph default
 be impossible becoming the / an actual primary.
 Typically you may be 
\emph on
already
\emph default
 in a split brain situation.
 This has not been observed for a long operations time on recent versions
 of MARS, but in general becoming primary via 
\family typewriter
--force
\family default
 cannot be guaranteed always, although MARS does its best.
 In split brain situations, or if you ever encounter such a problem, you
 
\emph on
must
\emph default
 resolve the split brain immediately after giving this command (see section
 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Split-Brain-Resolution"

\end_inset

).
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Hint in case of 
\begin_inset Formula $k>2$
\end_inset

 replicas: 
\family typewriter
marsadm invalidate
\family default
 cannot resolve a split brain at 
\emph on
other
\emph default
 secondaries (which are neither the old nor the new designated primary).
 Therefore, prefer the 
\family typewriter
leave-resource
\family default
 method described in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Split-Brain-Resolution"

\end_inset

, starting with a 
\family typewriter
leave-resource
\family default
 phase at the old primary, and proceeding to 
\begin_inset Quotes eld
\end_inset

unrelated
\begin_inset Quotes erd
\end_inset

 secondaries step by step, until the split brain is gone.
 Don't 
\family typewriter
join-resource
\family default
 again before the split brain is gone! This way, all these replicas will
 remain consistent for now, but of course outdated (or potentially even
 a 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Quotes erd
\end_inset

 split-brain version, but 
\emph on
potentially usable
\emph default
 in case you get under pressure in some way).
 In the hopefully unlikely case that you should later discover that you
 accidentally forced the 
\emph on
wrong
\emph default
 replica via 
\family typewriter
primary --force
\family default
, you will have a chance for recovery by either forcing the 
\begin_inset Quotes eld
\end_inset

correct
\begin_inset Quotes erd
\end_inset

 host to primary (if it did not already leave the resource), or by creating
 a completely fresh resource out of the 
\begin_inset Quotes eld
\end_inset

correct
\begin_inset Quotes erd
\end_inset

 local disk.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Generally: in case of 
\family typewriter
primary --force
\family default
, the preconditions are different.
 The fetch 
\emph on
must
\emph default
 be switched off (see 
\family typewriter
pause-fetch
\family default
), in order to get stable logfile positions.
 See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Forced-Switching"

\end_inset

.
 For your safety, 
\family typewriter
--force
\family default
 does not work in newer marsadm (after mars0.1stable52) when your replica
 is a current sync target.
 More explanations see section 
\begin_inset CommandInset ref
LatexCommand vref
reference "subsec:Forced-Switching"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
secondary
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
almost
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local 
\family typewriter
/dev/mars/$dev_name
\family default
 is no longer in use (e.g.
 umounted).
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: There exists no designated primary any more.
 During split brain and when the network is OK (again), all actual primaries
 (including the local host) will leave primary ASAP (i.e.
 when their 
\family typewriter
/dev/mars/mydata
\family default
 is no longer in use).
 Any secondary will start following (old) logfiles (even from backlogs)
 by replaying transaction logs if it is 
\emph on
uniquely
\emph default
 possible (which is often violated during split brain).
 On any secondary, 
\family typewriter
/dev/mars/$dev_name
\family default
 will have disappeared.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Notice: in difference to DRBD, you 
\series bold
don't need
\series default
 and you 
\series bold
should not use
\series default
 this command during normal operation, including handover.
 Any resource member which is 
\emph on
not
\emph default
 designated as primary will 
\emph on
automatically
\emph default
 go into secondary role.
 For example, if you have 
\begin_inset Formula $k=4$
\end_inset

 replicas, only 
\emph on
one of them
\emph default
 can be designated as a primary.
 When the network is OK, all other 3 nodes will know this fact, and they
 will 
\emph on
automatically
\emph default
 go into secondary mode, following the transaction logs from the (new) primary.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\size scriptsize
Hint: avoid this command.
 It turns off 
\emph on
any
\emph default
 primary, 
\series bold
globally
\series default

\begin_inset Foot
status open

\begin_layout Plain Layout

\size scriptsize
A serious 
\series bold
misconception
\series default
 among some people is when they believe that they can switch 
\begin_inset Quotes eld
\end_inset

a certain node to secondary
\begin_inset Quotes erd
\end_inset

.
 It is not possible to switch individual nodes to secondary, without affecting
 other nodes! The concept of 
\begin_inset Quotes eld
\end_inset

designated primary
\begin_inset Quotes erd
\end_inset

 is 
\series bold
global
\series default
 throughout a resource!
\end_layout

\end_inset

.
 You cannot start a sync after that (e.g.
 
\family typewriter
invalidate
\family default
 or 
\family typewriter
join-resource
\family default
 or 
\family typewriter
resume-sync
\family default
), because it is 
\emph on
not unique
\emph default
 wherefrom the data shall be fetched.
 In split brain situations (when the network is OK again), this may have
 further drawbacks.
 It is much better / easier to 
\series bold
\emph on
directly
\emph default
 switch the designated primary
\series default
 from one node to another via the 
\family typewriter
primary
\family default
 command.
 See also section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Forced-Switching"

\end_inset

.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 There is only one valid use case where you 
\emph on
really
\emph default
 need this command: before finally destroying a resouce via the 
\emph on
last
\emph default
 
\family typewriter
leave-resource
\family default
 (or before 
\emph on
forcefully killing
\emph default
 your resource via the dangerous 
\family typewriter
delete-resource
\family default
).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-umount
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Waiting"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
log-purge-all
\begin_inset CommandInset label
LatexCommand label
name "log-purge-all$res"

\end_inset


\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: all locally known logfiles and version links are removed,
 whenever they are not / no longer reachable by any split brain version.
\end_layout

\begin_layout Plain Layout
Rationale: remove hindering split-brain / 
\family typewriter
leave-resource
\family default
 leftovers.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Usually, you don't need this.
 
\family typewriter
leave-resource
\family default
 and 
\family typewriter
invalidate
\family default
 are already doing a similar logfile cleanup for you.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Use this only as a desperate last resort when split brain does not go away
 by means of 
\family typewriter
leave-resource
\family default
 (which 
\emph on
could
\emph default
 happen in very weird scenarios such as MARS running on virtual machines
 doing a restore of their snapshots, or otherwise unexpected resurrection
 of dead or half-dead nodes).
\end_layout

\begin_layout Plain Layout
THIS IS 
\emph on
POTENTIALLY
\emph default
 DANGEROUS
\end_layout

\begin_layout Plain Layout

\size scriptsize
This command 
\emph on
might
\emph default
 destroy some valuable logfiles / other information in case the local informatio
n is outdated or otherwise incorrect, as could be the case during very awkward
 disaster scenarios, such as corrupted 
\family typewriter
/mars
\family default
 filesystems.
 MARS does its best for checking anything, but there cannot be an absolute
 guarantee.
\end_layout

\begin_layout Plain Layout
That said, no single incident has been observed during millions of operation
 hours.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Hint: use 
\family typewriter
--dry-run
\family default
 beforehand for checking!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
err-purge-all
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: errors reported by 
\family typewriter
marsadm view $res --verbose
\family default
 are deleted after a while.
 However notice that some error reports may soon re-appear in case the error
 condition is persisting.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
link-purge-all
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: all deletable links withspeical value 
\family typewriter
.deleted
\family default
, which have been fully replicated throughout the whole cluster, will be
 deleted eventually.
 This is necessary to prevent inode overflow on 
\family typewriter
/mars
\family default
.
\end_layout

\begin_layout Plain Layout
Notice: 
\family typewriter
marsadm cron
\family default
 will do this also.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
resize
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

[$size]
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
almost
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: The local host must be primary.
 All disks in the cluster participating in 
\family typewriter
$res
\family default
 must be physically larger than the logical resource size (e.g, by use of
 
\family typewriter
lvm
\family default
; can be checked by macros 
\family typewriter
%disk-size{}
\family default
 and 
\family typewriter
%resource-size{}
\family default
).
 When the optional 
\family typewriter
$size
\family default
 argument is present, it must be smaller than the minimum of all physical
 sizes, but larger than the current logical size of the resource.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the logical size of 
\family typewriter
/dev/mars/$dev_name
\family default
 will reflect the new size after a while.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Logfile Operations
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="6" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
cron
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Do all necessary housekeeping tasks.
 See 
\family typewriter
log-rotate
\family default
 and 
\family typewriter
log-delete-all
\family default
 for details.
\end_layout

\begin_layout Plain Layout

\size scriptsize
This should be regularly called by an external cron job or similar.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
log-rotate
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node 
\family typewriter
$host
\family default
 must be primary at 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: after a while, a new transaction logfile 
\family typewriter
/mars/resource-$res/log-$new_nr-$host
\family default
 will be used instead of 
\family typewriter
/mars/resource-$res/log-$old_nr-$host
\family default
 where 
\family typewriter
$new_nr
\family default
 = 
\family typewriter
$old_nr
\family default
 + 1.
 Without 
\family typewriter
--force
\family default
, this will only carry out actions at the primary side since it makes no
 sense on secondaries.
 With 
\family typewriter
--force
\family default
, secondaries are 
\emph on
trying
\emph default
 to 
\emph on
remotely
\emph default
 trigger a log-rotate, but without any guarantee (likely even a split-brain
 may result instead, so use this only if you are 
\emph on
really
\emph default
 desperate).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
log-delete
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node must be a member of 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: when there exists some old transaction logfiles 
\family typewriter
/mars/resource-$res/log-*-$some_host
\family default
 which are no longer referenced by any of the symlinks 
\family typewriter
/mars/resource-$res/replay-*
\family default
 , those logfiles are marked for deletion in the whole cluster.
 When no such logfiles exist, nothing will happen.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
log-delete-one
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
Only useful for debugging.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node must be a member of 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: when there exists an old transaction logfile 
\family typewriter
/mars/resource-$res/log-$old_nr-$some_host
\family default
 where 
\family typewriter
$old_nr
\family default
 is the minimum existing number and that logfile is no longer referenced
 by any of the symlinks 
\family typewriter
/mars/resource-$res/replay-*
\family default
 , that logfile is marked for deletion in the whole cluster.
 When no such logfile exists, nothing will happen.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
log-delete-all
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Alias for 
\family typewriter
log-delete
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Consistency Operations
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="4" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
invalidate
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node must be in secondary role at 
\family typewriter
$res
\family default
.
 A 
\emph on
designated
\emph default
 primary must exist, and it must be reachable over network.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: the local disk is marked as 
\family typewriter
InConsistent
\family default
, and a fast fullsync from the designated primary will start after a while.
 Any 
\emph on
local
\emph default
 split brain (deviation from the designated primary) will be resolved, but
 any 
\emph on
other
\emph default
 split brain at other secondaries will 
\emph on
not
\emph default
 be affected.
 When the fullsync has finished successfully, the local node will be consistent
 again.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\size scriptsize
After a while, state 
\family typewriter
Orphan
\family default
 should be left.
 Don't forget to regularly monitor for longer occurrences of 
\family typewriter
Orphan
\family default
!
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
fake-sync
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node must be in secondary role at 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: when a fullsync is running, it will stop after a while, and
 the local node will be 
\emph on
marked
\emph default
 as consistent as if it were consistent again.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
THIS IS HIGLY DANGEROUS FOR DATA CONSISTENCY!
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
ONLY USE THIS IF YOU REALLY KNOW WHAT YOU ARE DOING!
\begin_inset Newline newline
\end_inset

See the WARNING in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Creating-and-Maintaining"

\end_inset


\begin_inset Newline newline
\end_inset

Use this only 
\emph on
before
\emph default
 creating a fresh filesystem inside 
\family typewriter
/dev/mars/$res
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-replay
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset


\size scriptsize
ONLY FOR ADVANCED HACKERS WHO KNOW WHAT THEY ARE DOING!
\begin_inset Newline newline
\end_inset

This command is deliberately not documented.
 You need the competence level RTFS (
\begin_inset Quotes eld
\end_inset

read the fucking sources
\begin_inset Quotes erd
\end_inset

).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Further 
\family typewriter
marsadm
\family default
 Operations
\begin_inset CommandInset label
LatexCommand label
name "sec:Further-marsadm-Operations"

\end_inset


\end_layout

\begin_layout Subsection
Inspection Commands
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="14" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
view-
\emph on
macroname
\begin_inset Newline newline
\end_inset


\emph default

\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Display the output of a macro evaluation.
 See section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Inspecting-the-State"

\end_inset

 for a thorough description.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
view
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Equivalent to 
\family typewriter
view-default
\family default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
role
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Use 
\family typewriter
view-role
\family default
 instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
state
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Use 
\family typewriter
view-state
\family default
 instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
cstate
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Use 
\family typewriter
view-cstate
\family default
 instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
dstate
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Use 
\family typewriter
view-dstate
\family default
 instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
status
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated.
 Use 
\family typewriter
view-status
\family default
 instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
show-state
\end_layout

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Don't use it.
 Use 
\family typewriter
view-state
\family default
 instead, or other macros.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
show-info
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Don't use it.
 Use 
\family typewriter
view-info
\family default
 instead, or other macros.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
show
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Don't use it.
 Implement your own macros instead.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
show-errors
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Deprectated, will vanish.
 Use 
\family typewriter
view-the-err-msg
\family default
 or 
\family typewriter
view-resource-err
\family default
 similar macros.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
cat
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$file
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Write the file content to stdout, but replace all occurences of numeric
 timestamps converted to a human-readable format.
 Thus is most useful for inspection of status and log files, e.g.
 
\family typewriter
marsadm cat /mars/5.total.log
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Setting Parameters
\begin_inset CommandInset label
LatexCommand label
name "subsec:Setting-Parameters"

\end_inset


\end_layout

\begin_layout Subsubsection
Per-Resource Parameters
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="4" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-emergency-limit $res 
\emph on
n
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
The argument 
\emph on
n
\emph default
 must be percentage between 0 and 100 %.
 When the remaining store space in 
\family typewriter
/mars/
\family default
 undershoots the given percentage, the resource will go 
\emph on
earlier
\emph default
 into emergency mode than by the global computation described in section
 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Defending-Overflow"

\end_inset

.
 0 means unlimited.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-emergency-limit $res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Inquiry of the preceding value.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsubsection
Global Parameters
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="9" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-sync-limit-value 
\emph on
n
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Limit the concurrency of sync operations to some maximum number.
 0 means unlimited.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-sync-limit-value
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Inquiry of the preceding value.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-connect-pref-list host1,host2,hostn
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Set the order of preferences for connections when there are more than 2
 hosts participating in a cluster.
 The argument must be comma-separated list of node names.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell multicolumn="1" alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-connect-pref-list
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Inquiry of the preceding value.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-global-enabled-log-compressions
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$features
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Tell the whole cluster which compression features to use globally for logfile
 compression.
 The effective value can be checked via 
\family typewriter
marsadm view-enabled-log-compressions
\family default
.
 See 
\family typewriter
marsadm view-potential-features
\family default
 and 
\family typewriter
marsadm --help
\family default
 for a list of compression feature names, which must be separated by 
\family typewriter
|
\family default
 symbols.
 Details are described in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:Data-Compression-Checksumming"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-global-enabled-net-compressions
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$features
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Tell the whole cluster which compression features to use globally for network
 transport compression.
 The effective value can be checked via 
\family typewriter
marsadm view-enabled-net-compressions
\family default
.
 See 
\family typewriter
marsadm view-potential-features
\family default
 and 
\family typewriter
marsadm --help
\family default
 for a list of compression feature names, which must be separated by 
\family typewriter
|
\family default
 symbols.
 Details are described in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:Data-Compression-Checksumming"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\end_inset

,
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-global-disabled-digests
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$features
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Tell the whole cluster which digests to disable globally for checksumming
 of transaction logfile data.
 The effective value can be checked via 
\family typewriter
marsadm view-disabled-digests
\family default
.
 See 
\family typewriter
marsadm view-potential-features
\family default
 and 
\family typewriter
marsadm --help
\family default
 for a list of compression feature names, which must be separated by 
\family typewriter
|
\family default
 symbols.
 Details are described in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:Data-Compression-Checksumming"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\end_inset

,
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Waiting
\begin_inset CommandInset label
LatexCommand label
name "subsec:Waiting"

\end_inset


\end_layout

\begin_layout Standard
For scripting, these commands are often needed for race avoidance.
 MARS' symlink are not propagated 
\emph on
immediately
\emph default
 throughout the whole cluster, but will take some time (also called 
\series bold
eventually consistent
\series default
 using a so-called Lamport clock).
 The following commands can be used for triggering a status update, and
 then waiting until information is recent enough.
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="6" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the 
\family typewriter
/mars/
\family default
 filesystem must be mounted and it must contain a valid MARS symlink tree
 produced by the other 
\family typewriter
marsadm
\family default
 commands.
 The kernel module must be loaded.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: none.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Wait until 
\emph on
all relevant 
\emph default
nodes in the cluster have sent a status update of their version of the symlink
 tree, or until timeout.
 The default timeout is 30 s (exceptionally) and may be changed by
\family typewriter
 --timeout=$seconds
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Use this for avoidance of rance conditions in the cluster, for nodes pariticipa
ting in some of the local resources.
 This command does its best to get the current status from the other cluster
 members.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\size scriptsize
This works only for resources which have been 
\emph on
already
\emph default
 joined.
 It you want to do a
\family typewriter
 join-resource
\family default
, use 
\family typewriter
update-cluster
\family default
 instead for fetching / updating the not-yet-joined symlink information
 also.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
update-cluster
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the 
\family typewriter
/mars/
\family default
 filesystem must be mounted everywhere and it must contain a valid MARS
 symlink tree produced by the other 
\family typewriter
marsadm
\family default
 commands.
 The network must be healthy.
 The kernel module must be loaded everywhere.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: none.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Wait until 
\emph on
all 
\emph default
nodes in the 
\emph on
whole
\emph default
 cluster have sent a status update of their 
\emph on
full
\emph default
 symlink tree, including any joined or non-joined information, or until
 timeout.
 The default timeout is 30 s (exceptionally) and may be changed by
\family typewriter
 --timeout=$seconds
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Use this before join-resource to ensure that 
\emph on
all
\emph default
 symlink information is recent.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-resource
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

{is-,}{attach,
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 primary,
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 device}{-off,}
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: the local node must be a member of the resource 
\family typewriter
$res
\family default
.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: none.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Wait until the local node reaches a specified condition on 
\family typewriter
$res
\family default
, or until timeout.
 The default timeout of 60 s may be changed by
\family typewriter
 --timeout=$seconds
\family default
.
 The last argument denotes the condition.
 The condition is inverted if suffixed by 
\family typewriter
-off
\family default
.
 When preceded by 
\family typewriter
is-
\family default
 (which is the most useful case), it is checked whether the condition is
 actually reached.
 When the 
\family typewriter
is-
\family default
 prefix is left off, the check is whether another 
\family typewriter
marsadm
\family default
 command has been already given which 
\emph on
tries
\emph default
 to achieves the intended result (typicially, you may use this after the
 
\family typewriter
is-
\family default
 variant has failed).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-connect
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
almost
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
This is an alias for 
\family typewriter
wait-cluster
\family default
 waiting until only those nodes are reachable which are particiapting at
 
\family typewriter
$res
\family default
 (instead of waiting for 
\emph on
all
\emph default
 hosts participating in 
\emph on
any
\emph default
 of the locally joined resources).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
wait-umount
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Precondition: none additionally.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Postcondition: 
\family typewriter
/dev/mars/$dev_name
\family default
 is no longer in use (e.g.
 umounted).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection

\family typewriter
systemd
\family default
 Control Commands
\begin_inset CommandInset label
LatexCommand label
name "subsec:systemd-Control-Commands"

\end_inset


\end_layout

\begin_layout Standard
These are optional commands when you want to use systemd for control of
 your services running on top of MARS, and for more automated handover /
 failover.
 The concept is described in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:systemd-Templates"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="6" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
systemd-trigger
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Tell the macro processor engine of marsadm that some systemd templates and/or
 configurations have changed.
 This will automatically re-compute any necessary systemd units residing
 in 
\family typewriter
/run/systemd/system/
\family default
 from template files provided in one of the directories from 
\family typewriter
@MARS_PATH/$SYSTEMD_SUBDIR/
\family default
, and will automatically remove any 
\emph on
instances
\emph default
 (not templates) which are no longer needed.
 Finally, 
\family typewriter
systemctl reload-daemon
\family default
 is executed, and any necessary units are started / stopped according to
 the new situation.
\begin_inset Newline newline
\end_inset

The macro language behind the template engine is described in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:systemd-Templates"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\size default

\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 When combined with 
\family typewriter
--force
\family default
, this will forcefully re-compute any template instances, even when already
 created before.
 Without 
\family typewriter
--force
\family default
, there is an internal check whether re-creation is necessary.
 This check might be fooled by manual intervention into 
\family typewriter
/run/systemd/system/
\family default
, so 
\family typewriter
--force
\family default
 will repair it.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-systemd-unit
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$start_unit
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$stop_unit
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Assign new start and unit names to a resource, or delete any existing names
 when the empty string 
\family typewriter

\begin_inset Quotes eld
\end_inset


\begin_inset Quotes erd
\end_inset


\family default
 is provided as 
\family typewriter
$start_unit
\family default
.
 Afterwards, a 
\family typewriter
systemd-trigger
\family default
 is executed, which causes start / stop operations according to the new
 situation.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-systemd-unit
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Report the current assignment of start and stop unit names to the given
 resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-systemd-want
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$hostname
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Manually override the host where $start_unit should appear.
 This is useful for a temporary stop of the applicatiom stack when $hostname
 has the special value 
\family typewriter

\begin_inset Quotes eld
\end_inset

(none)
\begin_inset Quotes erd
\end_inset


\family default
.
 See the 
\family typewriter
fsck 
\family default
example in section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "sec:systemd-Templates"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\begin_inset Newline newline
\end_inset

In split brain situations, it 
\emph on
might
\emph default
 be useful for manual override, but notice that 
\family typewriter
marsadm primary –force
\family default
 should be preferred because it switches 
\emph on
both
\emph default
 the designated primary 
\emph on
and
\emph default
 the application stack.
 Notice that the application stack won't actually start at a location where
 
\family typewriter
/dev/mars/$res
\family default
 is not present.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-systemd-want
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$res
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Report the current systemd start location for the given resource.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Low-Level Expert Commands
\end_layout

\begin_layout Standard
These commands are for experts and advanced sysadmins only.
 The interface is not stable, i.e.
 the meaning may change at any time.
 Use at your own risk!
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="4" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
set-link
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
RTFS.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-link
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
RTFS.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
delete-file
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
RTFS.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
The following commands are for manual setup / repair of cluster membership.
 Only to be used by experts who know what they are doing! In general, cluster-wi
de operations on IP addresses needs to be repeated at all hosts in the cluster
 iff the communication is not (yet) possible and/or not (yet) actually working
 (e.g.
 firewalling problems etc).
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="4" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "30col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
lowlevel-ls-host-ips
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "50col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
List all configured cluster members together with their currently configured
 IP addresses, as known in 
\family typewriter
/mars/ips/
\family default
 
\emph on
locally
\emph default
.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "30col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
lowlevel-set-host-ip
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$hostname
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$ip
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "50col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Change the assignment of IP addresses in 
\family typewriter
/mars/ips/
\family default
 at least 
\emph on
locally
\emph default
, and 
\emph on
try
\emph default
 to push the information also to other known peers (which is 
\emph on
unreliable
\emph default
 in general).
 This may be used when hosts are moved to different network locations, or
 when different network interfaces are to be used for replication (e.g.
 dedicated replication IPs).
 Notice that the names of hosts must not change at all, only their IP addresses
 may be changed, or new peers may be manually added this way.
\end_layout

\begin_layout Plain Layout

\size scriptsize
Tip: check active connections with 
\family typewriter
netstat
\family default
 & friends.
 Updates may need some time to proceed (socket timeouts etc).
\end_layout

\begin_layout Plain Layout

\size scriptsize
Hint: for safety, call this on 
\emph on
all
\emph default
 members of a cluster to ensure consistency.
 Otherwise it may happen that some cluster members do not know the 
\emph on
new
\emph default
 IP address where to fetch the 
\emph on
new
\emph default
 information from.
 See also the description of the 
\family typewriter
--ip-$peer=$peer_ip
\family default
 option.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\size scriptsize
For creation of new cluster memberships, always prefer 
\family typewriter
join-resource
\family default
.
 It checks for any uuid mismatches and for any resource name clashes / violation
s of resource name uniqueness, which could be
\series bold
 extremely dangerous
\series default
 for your data.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "30col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
lowlevel-delete-host
\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

$hostname
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "50col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Useful for decommissioning of dead / physically destroyed hosts after a
 disaster.
 Removes a host from the cluster membership in 
\family typewriter
/mars/ips/
\family default
 at least 
\emph on
locally
\emph default
, together with its IP address assignment.
 It also 
\emph on
tries
\emph default
 to push the deletion to other cluster members (which is 
\emph on
unreliable
\emph default
 in general).
 This does not remove any further information.
 In particular, resource memberships are untouched.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\size scriptsize
Please use 
\family typewriter
leave-resource --host=$name --force
\family default
 first.
 Repeat this for 
\emph on
all
\emph default
 former resource memberships.
 Otherwise you may produce left-over 
\begin_inset Quotes eld
\end_inset

zombie resource memberships
\begin_inset Quotes erd
\end_inset

, which in turm may prevent 
\family typewriter
marsadm cron
\family default
 from deleting logfile data, and consequently filling up 
\family typewriter
/mars/
\family default
 forever.
 When 
\emph on
all
\emph default
 replicas of any such resource are decommissioned eventually, also use 
\family typewriter
delete-resource --force
\family default
 and friends.
\end_layout

\begin_layout Plain Layout
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset


\size scriptsize
 Have a look at 
\family typewriter
leave-resource --host=$hostname
\family default
, first without 
\family typewriter
--force
\family default
, and also 
\family typewriter
leave-cluster --host=$hostname
\family default
, first without 
\family typewriter
--force
\family default
, which are checking for some common pitfalls.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Senseless Commands (from DRBD)
\end_layout

\begin_layout Standard
For completeness, here is a list of commands which do not make sense with
 MARS.
 Some of them are syntactically parsed for scrpting compatibility, but are
 not doing anything.
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="11" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
syncer
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
new-current-uuid
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
create-md
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
dump-md
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
dump
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
get-gi
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
show-gi
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
outdate
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
adjust
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
yes
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
Implemented as NOP (not necessary with MARS).
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
hidden-commands
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Subsection
Forbidden Commands (from DRBD)
\end_layout

\begin_layout Standard
These commands are not implemented because they would be dangerous in MARS
 context:
\end_layout

\begin_layout Standard

\size scriptsize
\begin_inset Tabular
<lyxtabular version="3" rows="3" columns="3">
<features islongtable="true" longtabularalignment="left">
<column alignment="left" valignment="top" width="0pt">
<column alignment="center" valignment="top">
<column alignment="left" valignment="top" width="0pt">
<row endhead="true" endfirsthead="true" endfoot="true" endlastfoot="true">
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Command / Params
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Cmp
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
Description
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
invalidate-remote
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="left" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
This would be dangerous in case you have multiple secondaries.
 A similar effect can be achieved with the 
\family typewriter
--host=
\family default
 option.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\family typewriter
\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "20col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\family typewriter
\size scriptsize
verify
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
no
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\size scriptsize
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "60col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout

\size scriptsize
This would cause unintended side effects due to races between logfile transfer
 / application and block-wise comparison of the underlying disks.
 However, 
\family typewriter
marsadm join-resource
\family default
 or 
\family typewriter
invalidate
\family default
 will do the same as DRBD verify followed by DRBD resync, i.e.
 this will automatically correct any found errors;.
 Note that the fast-fullsync algorithm of MARS will minimize network traffic.
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Chapter
Tuning, tips and tricks
\begin_inset CommandInset label
LatexCommand label
name "chap:Tuning,-tips-and"

\end_inset


\end_layout

\begin_layout Section
IO Performance Tuning
\begin_inset CommandInset label
LatexCommand label
name "sec:IO-Performance-Tuning"

\end_inset


\end_layout

\begin_layout Standard
There 
\emph on
exist
\emph default
 some use cases where MARS 
\emph on
can
\emph default
 deliver better IO performance than a raw block device.
 However, this cannot be expected 
\emph on
in general
\emph default
.
 In some 
\emph on
other
\emph default
 cases the performance may be 
\emph on
lower
\emph default
 than with a 
\emph on
single
\emph default
 local raw device.
\end_layout

\begin_layout Standard
For demonstration, we use the 
\family typewriter
blkreplay
\family default
 tool from 
\begin_inset Flex URL
status open

\begin_layout Plain Layout

http://blkreplay.org
\end_layout

\end_inset

 and a load which has been captured from a 
\series bold
real datacenter
\series default
 (1&1 Ionos ShaHoLin = Shared Hosting Linux).
 The load already contains a parallelism degree of 20 LXC containers running
 in parallel at the same iron.
 This corresponds to about 60,000 web spaces running on 20 Apache instances,
 already in parallel.
 In difference to artificial benchmarks (like pure random IO or pure sequential
 IO), this benchmark is much more close to real server operations, while
 artificial benchmarks are not meaningful for practice in general, because
 they can deviate from real server operations by 
\emph on
factors
\emph default
 or even by 
\series bold
orders of magnitude
\series default
.
\end_layout

\begin_layout Standard
In order to determine the limits of the test candidates, the timing of the
 original workload was converted to a linear ramp-up, simulating an 
\series bold
overloaded
\series default
 system.
 Otherwise benchmarking would not be possible.
\end_layout

\begin_layout Standard
The following 
\family typewriter
blkreplay
\family default
 benchmarks were executed on an otherwise unloaded Dell R630 with 40 CPU
 threads on 2 sockets, 192 GB RAM, a Dell R730 hardware RAID controller
 with 2 GB BBU cache, and 10 spindles Dell 1.8 TB 2.5 inch SAS disks configured
 as RAID-6.
 All data, including the 
\family typewriter
/mars
\family default
 directory, was located on the hardware RAID via LVM2.
 
\family typewriter
/dev/vginfong/lv-0
\family default
 was assigned a size of 8 TiB.
 For testing, vanilla kernel 4.9.x with the MARS pre-patch and 
\family typewriter
mars0.1astable72
\family default
 was used.
\end_layout

\begin_layout Standard
The 
\family typewriter
blkreplay
\family default
 parameters were as follows:
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

output_label="MARS"
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

# input description 
\end_layout

\begin_layout Plain Layout

input_file_list="http://blkreplay.org/loads/natural/1and1/shared-hosting/2016/Sha
HoLin_from_bare_metal/x20/shaholin-x20-ramped/shaholin-x20.adjacent.ramped-100.load.
gz"
\end_layout

\begin_layout Plain Layout

replay_duration=110
\end_layout

\begin_layout Plain Layout

speedup=10
\end_layout

\begin_layout Plain Layout

threads=512 
\end_layout

\begin_layout Plain Layout

cmode=with-conflicts
\end_layout

\begin_layout Plain Layout

scheduler="noop"
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

# hardware setup
\end_layout

\begin_layout Plain Layout

replay_host_list="icpu5133"
\end_layout

\begin_layout Plain Layout

replay_device_list="/dev/vginfong/lv-0"
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

# output description
\end_layout

\begin_layout Plain Layout

enable_graph=1
\end_layout

\begin_layout Plain Layout

graph_options="--no-static --dynamic" 
\end_layout

\end_inset


\end_layout

\begin_layout Standard
We start with the 
\series bold
raw
\series default
 device 
\family typewriter
/dev/vginfong/lv-0
\family default
 which had a size of 8 TiB.
 The throughput is about 1418 IOPS, and the latency diagram shows that the
 system is overloaded, but can cope with that overload: 
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/blkreplay/MARS.MARS.raw.iosched-noop.nr_request-128.icpu5133.vginfong.lv-0.g01.latency.realtime.png
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
As you can see in the filename, the NOOP kernel IO scheduler was used, and
 the kernel parameter 
\family typewriter
nr_requests
\family default
 was left at its default value of 128.
 When you read the specs of the Dell R730 hardware RAID controller, you
 will notice that it can handle a much higher IO request parallelism of
 almost 1024 requests in parallel.
\end_layout

\begin_layout Standard
So the first natural tuning attempt is 
\family typewriter
nr_requests=1020
\family default
, in order to release the 
\begin_inset Quotes eld
\end_inset

kernel IO handbrake
\begin_inset Quotes erd
\end_inset

.
 This results in an improved throughput of 1562 IOPS, and even the 
\emph on
maximum
\emph default
 latencies are improved, but the 
\emph on
average
\emph default
 latencies are becoming a little bit worse:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/blkreplay/MARS.MARS.raw.iosched-noop.nr_request-1020.icpu5133.vginfong.lv-0.g01.latency.realtime.png
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
It is well known since decades that there is a principal tradeoff between
 throughput and latencies in IO systems.
 Thus it is not a surprising result.
\end_layout

\begin_layout Standard
On servers, overload situations should be rare, and during overload throughput
 is typically much more important than latencies, as long as latencies are
 not exceedingly high.
 Thus we can recommend 
\family typewriter
nr_requests=1000
\family default
 for production.
\end_layout

\begin_layout Standard
However, some sysadmins might be tempted to question why the NOOP scheduler
 has been used.
 On the internet, there are a ton of claims that other IO schedulers like
 CFQ are much better.
\end_layout

\begin_layout Standard
Well, testing with CFQ instead of NOOP is no problem for 
\family typewriter
blkreplay
\family default
.
 However, the result is very surprising.
 While the IOPS are 1539, which is only a slight decrease which could result
 from measurement tolerances, the latencies are now turning almost into
 a disaster:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/blkreplay/MARS.MARS.raw.iosched-cfq.nr_request-1020.icpu5133.vginfong.lv-0.g01.latency.realtime.png
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
In production, you should never encounter IO latencies of almost 15 seconds.
 So what is going wrong here?
\end_layout

\begin_layout Standard
Here is an explanation.
 A hardware RAID controller 
\emph on
already
\emph default
 has an 
\emph on
internal
\emph default
 IO scheduler.
 This IO scheduler is hidden in a black box, such that many sysadmins don't
 know of its existence.
 If you add another IO scheduler at kernel level, you will have 
\series bold
two different
\series default
 IO schedulers running in parallel, and sometimes taking 
\series bold
contradictory decisions
\series default
.
\end_layout

\begin_layout Standard
These contradictory IO scheduling decisions may lead to problems in certain
 cases and scenarios.
\end_layout

\begin_layout Standard
There is another risk of interference with a third IO scheduler, which is
 MARS' internal asymmetric writeback scheduler.
 The latter is currently well-tuned for co-working with a BBU cache and
 its internal scheduler, running on bare metal.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Never use MARS inside of VMs! There you will have 
\emph on
several
\emph default
 additional IO schedulers and further types of IO bottlenecks
\begin_inset Foot
status open

\begin_layout Plain Layout
Example: data container formats like 
\family typewriter
qcow2
\family default
 can act as serious bottlenecks.
 Never place 
\family typewriter
/mars
\family default
 on top of them! Potential exceptions (after well-founded investigations)
 are functional testing, and certain non-critical workstation-like workloads.
 In general, never plan to place unknown or enterprise-critical workloads
 on top of them!
\end_layout

\end_inset

 inbetween, even if you try to disable some of them.
\end_layout

\begin_layout Standard
While kernel-level IO schedulers like CFQ certainly have their merits at
 improving your workstation's IO behaviour, they are counter-productive
 at servers with hardware RAID controllers.
\end_layout

\begin_layout Standard
So the advice is clear: 
\series bold
switch them off
\series default
 
\emph on
in such a case
\emph default
.
\end_layout

\begin_layout Standard
Even if you have a software RAID, check with 
\family typewriter
blkreplay
\family default
 that any IO schedulers are 
\emph on
really
\emph default
 improving things.
 Notice that device driver restrictions like 
\family typewriter
nr_requests
\family default
 may also work similarly to an IO scheduler.
 When possible, use your real workload, captured with 
\family typewriter
blktrace
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Never use a benchmark which only delivers IOPS! As demonstrated, inappropriate
 IOPS tuning (or choice of inappropriate components) can worsen latencies
 so much that production can be endangered!
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Always look at 
\emph on
both
\emph default
 IOPS 
\emph on
and
\emph default
 latencies!
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 
\emph on
Average
\emph default
 latencies, even when enriched with 
\emph on
standard deviation
\emph default
, are not enough.
 Classical statistics does not clearly describe operational problems like
 
\series bold
hangs
\series default
 and 
\series bold
exceptionally high latency requests
\series default
, which may occur only rarely, but can then lead to 
\series bold
serious incidents
\series default
.
 Use a tool which can clearly display 
\emph on
any
\emph default
 faulty behaviour, such as 
\family typewriter
blkreplay
\family default
's 
\series bold
latency diagrams
\series default
!
\end_layout

\begin_layout Standard
Now we come to benchmarking 
\family typewriter
/dev/mars/lv-0
\family default
 placed on top of 
\family typewriter
/dev/vginfong/lv-0
\family default
.
 Notice that MARS needs to write all write requests twice: once into the
 transaction logfile, and a second time by writeback into 
\family typewriter
/dev/vginfong/lv-0
\family default
.
\end_layout

\begin_layout Standard
So you might expect that performace of 
\family typewriter
/dev/mars/lv-0
\family default
 could be worse than at the underlying raw device.
\end_layout

\begin_layout Standard
Nevertheless, the 
\series bold
throughput
\series default
 is now measured 4338 IOPS, which means that performance has 
\series bold
more than doubled
\series default
.
 You can also see it by the duration of the benchmark at the x axis.
 Even the latencies have improved in many cases:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/blkreplay/MARS.MARS.mars.iosched-noop.nr_request-1020.icpu5133.mars.lv-0.g01.latency.realtime.png
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
How is it possible to be faster than a RAW device? How can this be explained?
\end_layout

\begin_layout Standard
Look at the graphics and at the explanations from section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:The-Transaction-Logger"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
 The key to local IO performance is the 
\series bold
re-ordering of writeback
\series default
 according to ascending sector numbers.
 This can reduce mechanical seek times of hard disks considerably, and even
 by factors, such that it can over-compensate the doubled writes to the
 transaction logfile, and even when both are residing at the same RAID set.
\end_layout

\begin_layout Standard
Since RAID-6 is has more expensive write operations due to its CRC computations
 and updates, RAID-6 will much more profit from this effect.
 Other RAID modes like RAID-10 may show a lower throughput improvement.
\end_layout

\begin_layout Standard
Notice: this effect is not only dependent from total RAM size and from the
 maximum size of the MARS temporary memory buffer (tuning parameter 
\family typewriter
/proc/sys/mars/mars_mem_percent
\family default
 which defaults to a limit of 20%).
 It is also highly dependent from the actual seek behaviour of the 
\series bold
workload
\series default
.
\end_layout

\begin_layout Standard
For example, if you use 
\family typewriter
dd
\family default
 for sequentially overwriting /dev/mars/lv-0 with a parallelism degree of
 1, the writeback optimization of MARS cannot be exploited.
 However, 
\family typewriter
dd
\family default
 is no appropriate benchmarking tool, and has almost nothing to do with
 real workloads occuring in datacenters, which typically are neither sequential,
 nor do they have a parallelism degree of only 1.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Please don't try to lead any discussions about this: simply use 
\family typewriter
blktrace
\family default
 to capture your real server workload, and compare it to a run of dd.
 Only if you encounter the same behaviour as 
\family typewriter
dd
\family default
, only then you can really claim that your workload is like 
\family typewriter
dd
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Any 
\series bold
assumptions about workloads
\series default
 are very dangerous: they can deviate from practice not only by factors,
 but sometimes even by 
\emph on
orders of magnitude
\emph default
.
 There no substitute for real measuments of 
\series bold
actual workload behaviour
\series default
.
\end_layout

\begin_layout Standard
Notice: the writeback optimization of MARS can typically only improve performanc
e of HDDs, but not of SSDs.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 By placing 
\family typewriter
/mars
\family default
 onto its own physical device with appropriate speed, you can compensate
 the doubled writes to some degree.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Depending on the workload and on RAID parameters, 
\family typewriter
/mars
\family default
 may be better placed onto SSDs, or better be placed on HDDs.
 There is no general rule.
 Just use 
\family typewriter
blktrace
\family default
 on your real workload, and check several configuration alternatives (also
 different RAID levels etc) with 
\family typewriter
blkreplay
\family default
.
\end_layout

\begin_layout Section
Data Compression and Checksumming (Digests)
\begin_inset CommandInset label
LatexCommand label
name "sec:Data-Compression-Checksumming"

\end_inset


\end_layout

\begin_layout Standard
Data compression can reduce the amount of data which needs to be piped through
 long-distance or other network bottlenecks.
 It is available in newer MARS versions, starting from mars0.1astable91.
 You also need to install the corresponding new version of 
\family typewriter
marsadm
\family default
 across the whole cluster.
\end_layout

\begin_layout Standard
The locally compiled-in compression and checksumming features as compiled
 into your currently running 
\family typewriter
mars.ko
\family default
 can be queried via
\end_layout

\begin_layout Quote

\family typewriter
marsadm view-implemented-features
\end_layout

\begin_layout Standard
The output may depend on your kernel compile options, such as the enabled
 crypto algorithms of your kernel.
 Typical output should look like
\end_layout

\begin_layout Quote
\begin_inset Flex URL
status open

\begin_layout Plain Layout

CHKSUM_MD5_OLD|CHKSUM_MD5|CHKSUM_CRC32C|CHKSUM_CRC32|CHKSUM_SHA1|COMPRESS_LZO|CO
MPRESS_LZ4|COMPRESS_ZLIB
\end_layout

\end_inset


\end_layout

\begin_layout Standard
In case you get less options, check your kernel 
\family typewriter
.config
\family default
 for the corresponding crypto algorithms, which can be compiled into your
 kernel firmly, or as a module.
 When necessary, re-compile your kernel with more crypto options enabled
 (see build instructions in section 
\begin_inset CommandInset ref
LatexCommand formatted
reference "sec:MARS-Kernel-Module"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
\end_layout

\begin_layout Standard
When the compile-time option 
\family typewriter
CONFIG_MARS_BENCHMARK=y
\family default
 is enabled, 
\family typewriter
modprobe mars
\family default
 will show you a list of benchmark results for each enabled crypto algorithm,
 in units of nanoseconds.
 Smaller numbers are better.
 Notice that results may depend on your processor model, and on availability
 of hardware acceleration (as supported by the crypto infrastucture of your
 kernel).
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Take the benchmark results with a grain of salt.
 The performance of some crypto algorithms may heavily depend on the 
\emph on
compressibility
\emph default
 of the data to be compressed.
 
\family typewriter
CONFIG_MARS_BENCHMARK
\family default
 uses a rather artifical test data pattern, which may deviate from the compressi
bility of your real productive data.
 Take the results with similar caution than BOGOMIPS, which are also not
 comparable with other benchmarks in general.
\end_layout

\begin_layout Standard
In order to work properly, 
\emph on
all
\emph default
 cluster members must have loaded a newer version of 
\family typewriter
mars.ko
\family default
.
 During rolling upgrade to newer MARS versions, mixed operation of different
 MARS versions is supported, even in combination with some old versions
 supporting only the traditional CHKSUM_MD5_OLD (which has some shortcomings
 and should not be used anymore in future).
 Only 
\emph on
common
\emph default
 features are actually usable.
 You can query the commonly usable options via the commands
\end_layout

\begin_layout Quote

\family typewriter
marsadm view-usable-compressions
\end_layout

\begin_layout Standard
and
\end_layout

\begin_layout Quote

\family typewriter
marsadm view-usable-digests
\end_layout

\begin_layout Standard
These should should show you a (possibly empty) list of those options which
 are really usable 
\emph on
at the moment
\emph default
.
 By installing newer / better versions of 
\family typewriter
mars.ko
\family default
 and 
\family typewriter
marsadm
\family default
, the list may become longer.
\end_layout

\begin_layout Standard
An overview of currently usable options, as well as the actually used algorithms
, are displayed at the headings produced by 
\family typewriter
marsadm view all
\family default
.
\end_layout

\begin_layout Subsection
Network Transport Compression
\begin_inset CommandInset label
LatexCommand label
name "subsec:Network-Transport-Compression"

\end_inset


\end_layout

\begin_layout Standard
By default, network transport compression is disabled, since it may worsen
 the CPU consumption.
 You can enabled it for the whole cluster via
\end_layout

\begin_layout Quote

\family typewriter
marsadm set-global-enabled-net-compressions
\backslash

\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 
\begin_inset Quotes eld
\end_inset

COMPRESS_LZO|COMPRESS_LZ4|COMPRESS_ZLIB
\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Standard
(or a shorter list of compress options), and you can disable it globally
 by supplying an empty list:
\end_layout

\begin_layout Quote

\family typewriter
marsadm set-global-enabled-net-compressions 
\begin_inset Quotes eld
\end_inset


\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Standard
Notice: this will compress the 
\emph on
data payloads
\emph default
 of network traffic, both for (incremental) logfile traffic (by default
 on port 7778), and for sync traffic (by default on port 7779).
\end_layout

\begin_layout Subsection
Logfile Payload Compression
\begin_inset CommandInset label
LatexCommand label
name "subsec:Logfile-Payload-Compression"

\end_inset


\end_layout

\begin_layout Standard
By default, logfile data compression is disabled, since it may worsen the
 CPU consumption, and may worsen local IO performance.
 You can enable it for the whole cluster via
\end_layout

\begin_layout Quote

\family typewriter
marsadm set-global-enabled-log-compressions
\backslash

\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 
\begin_inset Quotes eld
\end_inset

COMPRESS_LZO|COMPRESS_LZ4|COMPRESS_ZLIB
\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Standard
(or a shorter list of compress options), and you can disable it globally
 by supplying an empty list:
\end_layout

\begin_layout Quote

\family typewriter
marsadm set-global-enabled-log-compressions 
\begin_inset Quotes eld
\end_inset


\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Standard
In difference to network compression, this does not apply to sync data.
 It compresses the logfile payload 
\emph on
before
\emph default
 it is written to the transaction logfile.
 As a side effect, it also reduces network traffic, because the logfiles
 are usually smaller.
 Additionally, your 
\family typewriter
/mars
\family default
 directory may run out of space less quickly.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 However, as a major drawback, this may slow down the IO latencies of writes
 considerably, and thus may drastically reduce local IO performance (depending
 on performance of your crypto hardware, and on compressibility of data,
 etc).
 In particular, ZLIB is known to be a very slow algorithm (but to compress
 somewhat better than others), while LZO is a very old but very fast algorithm.
 In many cases, LZO or LZ4 are preferable.
 Do not enable this option blindly.
 Always observe the performance of your system afterwards.
\end_layout

\begin_layout Subsection
Logfile Payload Digests
\begin_inset CommandInset label
LatexCommand label
name "subsec:Logfile-Payload-Digests"

\end_inset


\end_layout

\begin_layout Standard
By default, all of these options are 
\emph on
enabled
\emph default
, because most users want to checksum the logfile data for detection of
 hardware errors, such as BBU cache failures, or silent corruption during
 the network transport of logfile data.
 When your secondaries encounter a checksum mismatch, they will 
\series bold
\emph on
refuse
\emph default
 to apply the defective data
\series default
, and will report 
\family typewriter
DefectiveLog
\family default
 in the 
\family typewriter
diskstate
\family default
 part of 
\family typewriter
marsadm view all
\family default
 (see section 
\begin_inset CommandInset ref
LatexCommand nameref
reference "subsec:Standard-marsadm-view"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
\end_layout

\begin_layout Standard
Most people view this behaviour as a 
\emph on
feature
\emph default
.
 It protects you from some types of data corruption.
\end_layout

\begin_layout Standard
If you want to disable some or all of the logfile digest algorithms, you
 can do via
\end_layout

\begin_layout Quote

\family typewriter
marsadm set-global-disabled-log-digests
\backslash

\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 
\begin_inset Quotes eld
\end_inset

CHKSUM_MD5_OLD|CHKSUM_MD5|CHKSUM_CRC32C|CHKSUM_CRC32|CHKSUM_SHA1
\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Disabling 
\emph on
all
\emph default
 of these options may improve local IO performance, but at the cost of less
 reliability.
 However, several compression algorithms are already doing some internal
 checksumming upon decompression.
 For maximum performance on weak hardware, it may pay off to enable compression,
 while disabling separate digesting.
 Please check what is the best combination for your hardware, your load,
 etc.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 If you decide to keep the logfile digests, e.g.
 when HA SLAs are more important than maximum performance: notice that checksumm
ing is done at the 
\emph on
input
\emph default
 data 
\emph on
before
\emph default
 any compression is applied.
 This increases safety against (potential / theoretical) problems with compressi
on / decompression errors.
\end_layout

\begin_layout Subsection
Network Payload Digests
\begin_inset CommandInset label
LatexCommand label
name "subsec:Network-Payload-Digests"

\end_inset


\end_layout

\begin_layout Standard
By default, all of these options are 
\emph on
enabled
\emph default
, because checksumming over the network at fast full-sync cannot be disabled
 by concept.
 At least one of the network digests must always remain enabled.
 If you try to disable all of them, an automatic fallback to
\family typewriter
 CHKSUM_MD5_OLD
\family default
 will occur.
 Since this a rather slow and non-optimum algorithm, disabling the faster
 ones (such as 
\family typewriter
CHKSUM_CRC32C
\family default
) is no good idea.
\end_layout

\begin_layout Standard
If you want to disable some of the network digest algorithms, you can do
 similarly to
\end_layout

\begin_layout Quote

\family typewriter
marsadm set-global-disabled-net-digests
\backslash

\begin_inset Newline newline
\end_inset


\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
strut
\backslash
hfill
\end_layout

\end_inset

 
\begin_inset Quotes eld
\end_inset

CHKSUM_MD5|CHKSUM_CRC32C|CHKSUM_CRC32|CHKSUM_SHA1
\begin_inset Quotes erd
\end_inset


\end_layout

\begin_layout Section
The 
\family typewriter
/proc/sys/mars/
\family default
 and other Expert Tweaks
\begin_inset CommandInset label
LatexCommand label
name "sec:The-/proc/sys/mars/-Expert"

\end_inset


\end_layout

\begin_layout Standard
In many cases, you will not need to deal with tweaks in 
\family typewriter
/proc/sys/mars/
\family default
 because everything should already default to reasonable predefined values.
 This is not a 
\begin_inset Quotes eld
\end_inset

stable
\begin_inset Quotes erd
\end_inset

 interface.
 It may change during development of MARS.
 It allows access to some 
\emph on
internal
\emph default
 kernel variables of the 
\family typewriter
mars.ko
\family default
 kernel module at 
\emph on
runtime
\emph default
.
\end_layout

\begin_layout Standard
This means, all values modified via /proc/ are not persistent.
 They will be reset to default at 
\family typewriter
rmmod mars
\family default
 or at reboot.
 If you need some persistence, implement it by yourself, e.g.
 at startup scripts.
\end_layout

\begin_layout Standard
This section describes only those tweaks which could be helpful for sysadmins,
 but not those for developers / very deep internals.
\end_layout

\begin_layout Subsection
Tuning Network Performance
\begin_inset CommandInset label
LatexCommand label
name "subsec:Tuning-Network-Performance"

\end_inset


\end_layout

\begin_layout Standard
Since a few years, a feature called 
\begin_inset Quotes eld
\end_inset

socket bundling
\begin_inset Quotes erd
\end_inset

 is available.
\end_layout

\begin_layout Standard
It is mostly intendend for lines showing high packet loss.
 By using multiple TCP sockets in parallel for emulating a single logical
 connection, throughput can be significantly increased.
\end_layout

\begin_layout Standard
Example for setting the socket parallelism to 4:
\end_layout

\begin_layout Itemize

\family typewriter
echo 4 > /proc/sys/mars/parallel_connections
\end_layout

\begin_layout Standard
The following graphics shows the throughput of a non-fast
\begin_inset Foot
status open

\begin_layout Plain Layout
The fast fullsync algorithm would not saturate the 
\family typewriter
eth0
\family default
 link with traffic from a single resource.
\end_layout

\end_inset

 fullsync of a 
\emph on
single
\emph default
 100GiB resource over a loaded long-distance line between Europe/Germany
 and USA/Midwest.
 In order to compensate highly varying load at the line, all the experiments
 were repeated more than 10 times and averaged.
 Each bar shows the throughput for a particular socket parallelism.
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/socket-bundling-long-summary.png
	width 70col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
Notice that the uplinks of the two servers are only 1 GBit/s respectively.
 When the uplink is saturated, about 100 MByte/s is the maximum possible
 peak throughput in theory.
 You can easily recognize that the peak throughput is almost reached with
 a parallelism degree of 2, but using even more sockets appears to be slightly
 counter-productive.
 One of the reasons is that more sockets will increase contention on the
 line, and thus increasing packet loss.
 Another potential reason is that higher parallelism at sockets will lead
 to higher parallelism in disk reads, in turn leading to more permutations
 of disk read positions (more 
\emph on
random
\emph default
 reads instead of purely sequential reads), which is counter-productive
 for disk readahead strategies.
\end_layout

\begin_layout Standard
The next graphics shows the same, but over a medium distance of about 50km.
 This line is even more heavily loaded with respect to the number of TCP
 connections running in parallel (probly some 10,000 or even 100,000 if
 not more), and there is some kind of 
\begin_inset Quotes eld
\end_inset

traffic shaping
\begin_inset Quotes erd
\end_inset

 at some intermediate network gear which will 
\begin_inset Quotes eld
\end_inset

punish
\begin_inset Quotes erd
\end_inset

 those traffic sources disproportionally increasing overall packet loss.
 This can explain the even higher counter-productive effect of using too
 much sockets and thus injecting additional packet loss:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/socket-bundling-short-summary.png
	width 70col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

In general, the optimum value for 
\family typewriter
/proc/sys/mars/parallel_connections
\family default
 may depend on many runtime factors such as other load running over some
 (parts of) physical equipment.
 You will need to determine optimum values yourself.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Notice that socket bundling is conceptually the 
\begin_inset Quotes eld
\end_inset

opposite
\begin_inset Quotes erd
\end_inset

 of traffic shaping.
 You are trying to get 
\emph on
more
\emph default
 bandwidth, at the cost of 
\emph on
other
\emph default
 traffic competing for the same network resources.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 If you are operating masses of servers, don't set the MARS socket parallelism
 
\series bold
too high 
\series default
everywhere.
 You might 
\begin_inset Quotes eld
\end_inset

steal
\begin_inset Quotes erd
\end_inset

 too much bandwidth from other applications when starting masses of syncs
 in parallel, e.g.
 after an incident.
 Best practice is to start with a default value of 1, and to increase it
 only 
\emph on
on demand
\emph default
, and/or preferably 
\emph on
only
\emph default
 at those servers where high load really occurs or where some urgent actions
 need a 
\emph on
temporary
\emph default
 boost.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Experts in networking may try to load-balance the parallel TCP connections
 over multiple physical paths, for example by hashing over the dynamic source
 port numbers.
 However, we currently have no experience with suchalike setups.
\end_layout

\begin_layout Subsection
Syslogging
\end_layout

\begin_layout Standard
All internal messages produced by the kernel module belong to one of the
 following classes:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
0 debug messages
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
1 info messages
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
2 warnings
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
3 error messages
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
4 fatal error messages
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
5 any message (summary of 0 to 4)
\end_layout

\begin_layout Subsubsection
Logging to Files
\end_layout

\begin_layout Standard
This feature will likely disappear when MARS goes to kernel upstream.
 It was mostly intended for debugging during early beta phases and is no
 longer needed for stable operation.
 Developers may use it for spotting potential problems.
\end_layout

\begin_layout Standard
The classes may be used to produce status files 
\family typewriter
$class.*.status
\family default
 in the 
\family typewriter
/mars/
\family default
 and/or in the 
\family typewriter
/mars/resource-
\emph on
mydata
\emph default
/
\family default
 directory / directories.
\end_layout

\begin_layout Standard
When you create a file 
\family typewriter
$class.*.log
\family default
 in parallel to any 
\family typewriter
$class.*.status
\family default
, the 
\family typewriter
*.log
\family default
 file will be appended forever with the same messages as in 
\family typewriter
*.status
\family default
.
 The difference is that *.status is regenerated anew from an empty starting
 point, while *.log can (potentially) increase indefinitely unless you remove
 it, or rename it to something else.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Beware, any permamently present 
\family typewriter
*.log
\family default
 file can easily fill up your 
\family typewriter
/mars/
\family default
 partition until the problems described in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Defending-Overflow"

\end_inset

 will appear.
 Use 
\family typewriter
*.log
\family default
 only for a 
\series bold
limited time
\series default
, and 
\series bold
only for debugging!
\end_layout

\begin_layout Subsubsection
Logging to Syslog
\end_layout

\begin_layout Standard
The classes also play a role in the following 
\family typewriter
/proc/sys/mars/
\family default
 tweaks:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
syslog_min_class
\family default
 (rw) The 
\emph on
mimimum
\emph default
 class number for 
\emph on
permanent
\emph default
 syslogging.
 By default, this is set to -1 in order to switch off perment logging completely.
 Permament logging can easily flood your syslog with such huge amounts of
 messages (in particular when class=0), that your system as a whole may
 become unusable (because vital kernel threads may be blocked too long or
 too often by the userspace syslog daemon).
 Instead, please use the flood-protected syslogging described below!
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
syslog_max_class
\family default
 (rw) The 
\emph on
maximum
\emph default
 class number for 
\emph on
permanent
\emph default
 syslogging.
 Please use the flood-protected version instead.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
syslog_flood_class
\family default
 (rw) The mimimum class of flood-protected syslogging.
 The maximum class is always 4.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
syslog_flood_limit
\family default
 (rw) The maxmimum number of messages after which the flood protection will
 start.
 This is a hard limit for the the number of messages written to the syslog.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
syslog_flood_recovery_s
\family default
 (rw) The number of seconds after which the internal flood counter is reset
 (after flood protection state has been reached).
 When no new messages appear after this time, the flood protection will
 start over at count 0.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

The rationale behind flood protected syslogging: sysadmins are usually only
 interested in the point in time where some problems / incidents / etc have
 
\emph on
started
\emph default
.
 They are usually not interested in capturing 
\emph on
each
\emph default
 and 
\emph on
every
\emph default
 single error message (in particular when they are flooding the system logs).
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

If you 
\emph on
really
\emph default
 need complete error information, use the 
\family typewriter
*.log
\family default
 files described above, compress them and save them to somewhere else 
\emph on
regularly
\emph default
 by a cron job.
 This bears much less overhead than filtering via the syslog daemon, or
 even remote syslogging in real time which will almost surely screw up your
 system in case of network problems co-inciding with flood messages, such
 as caused in turn by those problems.
 Don't rely on real-time concepts, just do it the old-fashioned batch job
 way.
\end_layout

\begin_layout Subsubsection
Tuning Verbosity of Logging
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
show_debug_messages
\family default
 Boolean switch, 0 or 1.
 Mostly useful only for kernel developers.
 This can easily flood your logs if our are not careful.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
show_log_messages
\family default
 Boolean switch, 0 or 1.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
show_connections
\family default
 Boolean switch, 0 or 1.
 Show detailed internal statistics on sockets.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
show_statistics_local
\begin_inset space ~
\end_inset

/
\begin_inset space ~
\end_inset

show_statistics_global
\family default
 Only useful for kernel developers.
 Shows some internal information on internal brick instances, memory usage,
 etc.
\end_layout

\begin_layout Subsection
Tuning the Sync
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
sync_flip_interval_sec
\family default
 (rw) The sync process must not run in parallel to logfile replay, in order
 to easily guarantee consistency of your disk.
 If logfile replay would be paused for the full duration of very large or
 long-lasting syncs (which could take some days over very slow networks),
 your 
\family typewriter
/mars/
\family default
 filesystem could overflow because no replay would be possible in the meantime.
 Therefore, MARS regulary flips between actually syncing and actually replaying,
 if both is enabled.
 You can set the time interval for flipping here.
\begin_inset Newline newline
\end_inset

Increasing this value may improve overall sync throughput, at the cost of
 some more space required by 
\family typewriter
/mars
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
sync_limit
\family default
 (rw) When > 0, this limits the maximum number of sync processes actually
 running parallel.
 This is useful if you have a large number of resources, and you don't want
 to overload the network and/or your local IO system with too many sync
 processes running 
\emph on
in parallel
\emph default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
sync_nr
\family default
 (ro) Passive indicator for the number of sync processes currently running.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
sync_want
\family default
 (ro) Passive indicator for the number of sync processes which 
\emph on
demand
\emph default
 running.
\end_layout

\begin_layout Subsection
Lowlevel TCP Tuning (Networking Experts Only)
\begin_inset CommandInset label
LatexCommand label
name "subsec:TCP-Tuning"

\end_inset


\end_layout

\begin_layout Standard
When 
\family typewriter
CONFIG_MARS_SEPARATE_PORTS
\family default
 and 
\family typewriter
CONFIG_MARS_IPv4_TOS
\family default
 are enabled, MARS uses the following types of traffic:
\end_layout

\begin_layout Description

\family typewriter
MARS_TRAFFIC_META
\family default
 (by default on port 7777 with 
\family typewriter
IPTOS_LOWDELAY
\family default
) This can be tuned in directory 
\family typewriter
/proc/sys/mars/tcp_tuning_0_meta_traffic/
\family default
.
\end_layout

\begin_layout Description

\family typewriter
MARS_TRAFFIC_REPLICATION
\family default
 (by default on port 7778 with 
\family typewriter
IPTOS_RELIABILITY
\family default
) This can be tuned in directory 
\family typewriter
/proc/sys/mars/tcp_tuning_1_replication_traffic/
\family default
.
\end_layout

\begin_layout Description

\family typewriter
MARS_TRAFFIC_SYNC
\family default
 (by default on port 7779 with 
\family typewriter
IPTOS_MINCOST
\family default
) This can be tuned in directory 
\family typewriter
/proc/sys/mars/tcp_tuning_2_sync_traffic/
\family default
.
 Attention: since the advent of 
\family typewriter
DSCP
\family default
, this bit (hex 
\family typewriter
0x2
\family default
 in host byte order) is suppressed by the kernel, and yields 
\family typewriter
DS0
\family default
.
\end_layout

\begin_layout Standard
In each of these directories, the following tunables are available (only
 for networking experts who know what they are doing):
\end_layout

\begin_layout Description

\family typewriter
ip_tos
\family default
 As explained above.
 Notice: hex constants from 
\family typewriter
/usr/include/linux/ip.h
\family default
 must be converted to decimal before forwarding to the 
\family typewriter
/proc
\family default
 interface.
\end_layout

\begin_layout Description

\family typewriter
tcp_window_size
\family default
 Current default is 8 * 1024 * 1024.
\end_layout

\begin_layout Description

\family typewriter
tcp_nodelay
\family default
 Current default is 0.
\end_layout

\begin_layout Description

\family typewriter
tcp_timeout
\family default
 Current default is 2.
\end_layout

\begin_layout Description

\family typewriter
tcp_keepcnt
\family default
 Current default is 3.
\end_layout

\begin_layout Description

\family typewriter
tcp_keepintvl
\family default
 Current default is 3.
\end_layout

\begin_layout Description

\family typewriter
tcp_keepidle
\family default
 Current default is 4.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Further tuning parameters are in the standard Linux kernel.
 Notice that 
\family typewriter
IP_TOS
\family default
 is internally converted to 
\family typewriter
DSCP
\family default
, which in turn can be further manipulated by 
\family typewriter
netfilter
\family default
 / 
\family typewriter
iptables
\family default
 and/or by 
\family typewriter
qdisc
\family default
 (
\family typewriter
tc
\family default
) and/or by further (external) networking components.
 The ancient TOS settings are meant as a default 
\emph on
starting point
\emph default
 for further customization to your needs.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Typically, 
\emph on
public
\emph default
 internet transports are flattening / ignoring or otherwise manipulating
\begin_inset Foot
status open

\begin_layout Plain Layout
DSCP markings can be only made reliable on private networks (possibly requiring
 some effort).
 Public Internet service and transit providers do not necessarily treat
 the TOS values or DSCP markings with any form of priority and may also
 remove or change them without any notice.
 Some internet service or transit providers also do use specific DSCP markings
 to mark packets for being dropped, which may result in hard to find transmissio
n errors.
\end_layout

\begin_layout Plain Layout
If want to use MARS on a public internet connection, you should use 
\series bold
encrypted
\series default
 
\series bold
VPN
\series default
 with different DSCP markings, and coordinate them with your network services
 provider.
\end_layout

\end_inset

 the TOS / DSCP fields.
 There it will not work.
 Anyway, you should never route unencrypted MARS traffic over public transports,
 for obvious security reasons.
 Notice: MARS replication is meant for company-
\emph on
internal
\emph default
 networks like 
\emph on
internal
\emph default
 
\series bold
replication networks
\series default
 (or storage networks) where some networking department has control of.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Playing with the above settings can easily tear down your whole (replication)
 network if you don't know exactly what you are doing.
 Please test any changes in the lab first.
 Mass rollout should be done in incremental phases, each in power of 10
 units.
 There might be unexpected effects like packet storms, or packet loss, etc.
 Some of these effects may only show up when a certain number of hosts is
 exceeded, or when certain load conditions are hammering the overall Distributed
 System.
 Some very old routers / switches are known to break down unexpectedly when
 overloaded in certain ways.
 Be careful in a production environment!
\end_layout

\begin_layout Chapter
Advanced users: automation and the macro processor
\begin_inset CommandInset label
LatexCommand label
name "chap:Advanced-users:-automation"

\end_inset


\end_layout

\begin_layout Section
The 
\family typewriter
systemd
\family default
 Template Interface
\begin_inset CommandInset label
LatexCommand label
name "sec:systemd-Templates"

\end_inset


\end_layout

\begin_layout Standard
Starting with 
\family typewriter
mars0.1astable
\family default
79, you may use 
\family typewriter
systemd
\family default
 as a cluster manager at the Mechanics Layer.
 MARS will replicate any 
\family typewriter
systemd
\family default
-relevant state information across the (big) cluster, achieving some remote
 control.
 In particular, automated handover via 
\family typewriter
marsadm primary $resource
\family default
 is supported.
 More features will likely be added to future releases.
\end_layout

\begin_layout Subsection
Why 
\family typewriter
systemd
\family default
?
\end_layout

\begin_layout Standard
All major Linux distributions are now 
\family typewriter
systemd
\family default
 based.
 It is the new quasi standard.
 Although there have been some discussions in the community about its merits
 and its shortcomings, now it appears to be accepted in large parts of the
 Linux world.
\end_layout

\begin_layout Standard
Systemd has a few advantages:
\end_layout

\begin_layout Enumerate
It is running as 
\family typewriter
init
\family default
 process under the reserved 
\family typewriter
pid=1
\family default
.
 If it ever would die, then your system as a whole would die.
 There is no need for adding a new MARS clustermanager daemon or similar,
 which could fail independently from other parts of the system.
\end_layout

\begin_layout Enumerate
Although 
\family typewriter
systemd
\family default
 has been criticised as being 
\begin_inset Quotes eld
\end_inset

monolithic
\begin_inset Quotes erd
\end_inset

 (referring to its internal software architecture), its 
\emph on
usage
\emph default
 by sysadmins is easily decomposable into many plugins called 
\series bold
units
\series default
.
\end_layout

\begin_layout Enumerate
Local LXC containers, local VMs, 
\family typewriter
iSCSI
\family default
 exports, 
\family typewriter
nfs
\family default
 exports and many other parts of the system are often already controlled
 by 
\family typewriter
systemd
\family default
.
 Together with 
\family typewriter
udev
\family default
 and other parts, it already controls devices, LVM, mountpoints, etc.
 Since MARS is only a particular 
\emph on
component
\emph default
 in a bigger complicated stack, it is an advantage to use the same (more
 or less standardized and well-integrated) tools for managing the whole
 stack.
\end_layout

\begin_layout Standard
Systemd has also a few disadvantages:
\end_layout

\begin_layout Enumerate
It is not accepted everywhere.
 Therefore the 
\family typewriter
systemd
\family default
 template extensions of 
\family typewriter
marsadm
\family default
 are not mandatory for MARS operations.
 You can implement your own alternatives when necessary.
\end_layout

\begin_layout Enumerate
It can be messy to deal with.
 In particular, it can sometimes 
\emph on
believe
\emph default
 that the system 
\emph on
were
\emph default
 in a particular state, although in reality it isn't.
 Compensation is hairy.
\end_layout

\begin_layout Enumerate
Usablility / reporting: it is less usable for getting an overview over a
 bigger local system, and is practically unusable (out-of-the-box) for managing
 a bigger cluster at cluster level.
 Monitoring needs to be done separately.
\end_layout

\begin_layout Subsection
Working Principle of the 
\family typewriter
systemd
\family default
 Template Engine
\begin_inset CommandInset label
LatexCommand label
name "subsec:Working-Principle-of"

\end_inset


\end_layout

\begin_layout Standard
Systemd already has some very basic templating capabilities.
 It is possible to create unit names containing the 
\family typewriter
@
\family default
 symbol, which can then be expanded under certain circumstances, e.g.
 to tty names etc.
 However, automatic expansion is only done when somebody knows the instance
 name already 
\emph on
in advance
\emph default
.
 The author has not found any way for creating instance names out of 
\begin_inset Quotes eld
\end_inset

thin air
\begin_inset Quotes erd
\end_inset

, such as from dynamically created MARS resource names.
 Essentially, an 
\emph on
inference machine
\emph default
 for systemd templates does not yet exist.
\end_layout

\begin_layout Standard
This lacking functionality is completed with the following macro processing
 capabilities of 
\family typewriter
marsadm
\family default
 (see section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:The-macro-processor"
plural "false"
caps "false"
noprefix "false"

\end_inset

):
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
sloppy
\end_layout

\end_inset

 Some ordinary or templated 
\family typewriter
systemd
\family default
 unit files (see 
\family typewriter
man systemd.unit
\family default
) can be installed into one of the following directories: 
\family typewriter
./systemd-templates
\family default
, 
\family typewriter
$HOME/.marsadm/systemd-templates/
\family default
, 
\family typewriter
/etc/marsadm/systemd-templates/
\family default
, 
\family typewriter
/usr/lib/marsadm/systemd-templates/
\family default
, 
\family typewriter
/usr/local/lib/marsadm/systemd-templates/
\family default
.
 Futher places can be defined by overriding the $
\family typewriter
MARS_PATH
\family default
 environment variable.
\end_layout

\begin_layout Standard
From these directories, ordinary systemd unit files will be just copied
 into 
\family typewriter
/run/systemd/system/
\family default
 (configurable via 
\family typewriter
$SYSTEMD_TARGET_DIR
\family default
) and then picked up by 
\family typewriter
systemd
\family default
 as ordinary unit files.
\end_layout

\begin_layout Standard
Template unit files are nothing but unit files, optionally containing 
\family typewriter
@{
\emph on
varname
\emph default
}
\family default
 parts or other macro definitions in their filename, and possibly also in
 their bodies, at arbitrary places.
 These 
\family typewriter
@{...}
\family default
 parts are substituted by a 
\family typewriter
marsadm
\family default
 macro processing engine.
\end_layout

\begin_layout Standard
The following macro capabilities are currently defined:
\end_layout

\begin_layout Description

\family typewriter
@{
\emph on
varname
\emph default
}
\family default
 Expands to the value of the variable.
 This can be used both in template filenames and and in content of template
 files.
 Predefined are the following variables:
\end_layout

\begin_deeper
\begin_layout Description

\family typewriter
@{res}
\family default
 The MARS resource name.
\end_layout

\begin_layout Description

\family typewriter
@{resdir}
\family default
 The MARS resource directory 
\family typewriter
/mars/resource-$res/
\family default
.
\end_layout

\begin_layout Description

\family typewriter
@{host}
\family default
 The local host name as determined by 
\family typewriter
marsadm
\family default
, or as overridden by the 
\family typewriter
--host=
\family default
 parameter.
\end_layout

\begin_layout Description

\family typewriter
@{cmd}
\family default
 The 
\family typewriter
marsadm
\family default
 command as given on the command line (only reasonable for debugging or
 for error messages).
\end_layout

\begin_layout Description

\family typewriter
@{
\emph on
varname
\emph default
}
\family default
 Further variables as defined by the macro processor, see section 
\begin_inset CommandInset ref
LatexCommand vref
reference "par:Predefined-Variables"

\end_inset

, and as definable by 
\family typewriter
%let{
\emph on
varname
\emph default
}{...}
\family default
 statements, see also section 
\begin_inset CommandInset ref
LatexCommand vref
reference "subsec:Predefined-Trivial-Macros"

\end_inset

.
\end_layout

\end_deeper
\begin_layout Description

\family typewriter
@eval{
\emph on
text
\emph default
}
\family default
 Calls the MARS macro processor as explained in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:The-macro-processor"
plural "false"
caps "false"
noprefix "false"

\end_inset

, and substitutes its output.
 Notice that systemd template variables occurring in the macro processor
 
\family typewriter
\emph on
text
\family default
\emph default
 must be accessed via the macro processor syntax 
\family typewriter
%{varname}
\family default
, because the macro processor uses 
\family typewriter
%
\family default
 as an escape symbol, while the systemd template engine uses 
\family typewriter
@
\family default
 instead.
 This is necessary for distinction of both layers.
 Notice that variables defined via the macro processor syntax 
\family typewriter
%let{varname}{value}
\family default
 can be afterwards accessed by the template engine via 
\family typewriter
@{varname}
\family default
 syntax, once the macro engine has finished working on 
\family typewriter
\emph on
text
\family default
\emph default
.
\end_layout

\begin_layout Description

\family typewriter
^
\emph on
{varname
\emph default
}
\family default
 This can be used in template filenames 
\emph on
only
\emph default
.
 First, any 
\family typewriter
@{
\emph on
othername
\emph default
}
\family default
 are substituted.
 Finally, any 
\family typewriter
^
\emph on
{varname
\emph default
}
\family default
 are matched against the actual filename like a shell wildcard 
\family typewriter
*
\family default
.
 The matching part of the filename is assigned to 
\family typewriter
\emph on
varname
\family default
\emph default
, and can be later used at 
\family typewriter
@{
\emph on
varname
\emph default
}
\family default
 substitutions occurring in the 
\emph on
content
\emph default
 of the file.
\end_layout

\begin_layout Description

\family typewriter
@esc{
\emph on
text
\emph default
}
\family default
 Calls the 
\family typewriter
systemd-escape
\family default
 tool for conversion of pathnames following the 
\family typewriter
systemd
\family default
 naming conventions (see 
\family typewriter
man systemd-escape
\family default
).
 For example, a dash is converted to 
\family typewriter

\backslash
x2d
\family default
.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Omission of 
\family typewriter
systemd-escape
\family default
 can lead to problems when your resource names are containing special characters
 like dashes or other special symbols (in the sense of 
\family typewriter
systemd
\family default
).
 Bugs of this kind are hard to find and to debug.
 Either forbid special characters in your installation, or don't forget
 to test everything with some crude resource names!
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Example snippet from a 
\family typewriter
.path
\family default
 unit.
 Please notice where escaping is needed and where it must not be used (also
 notice that a dash is sometimes a legal part of the 
\family typewriter
.mount
\family default
 unit name, but except from the resource name part):
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

[Path]
\end_layout

\begin_layout Plain Layout

PathExists=/dev/mars/@{res}
\end_layout

\begin_layout Plain Layout

Unit=vol-@escvar{res}.mount 
\end_layout

\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Another source of crude bugs is the backslash character in the 
\family typewriter
systemd-escape
\family default
 substitution, such as from 
\family typewriter

\backslash
x2d
\family default
.
 When passed to a shell, such as in certain 
\family typewriter
ExecStart=
\family default
 statements like 
\family typewriter
/bin/bash -c
\series bold
 $args
\family default
\series default
, the backslash will be removed.
 Therefore, don't forget to either replace any single backslash with two
 backslashes, or to put the whole pathname in single quotes, or similar.
 Always check the result of your substitutions! It depends on the 
\emph on
target
\emph default
 (such as 
\family typewriter
bash
\family default
, as opposed to 
\family typewriter
systemd
\family default
) whether further escaping of the escapes is needed, or whether it 
\emph on
must not
\emph default
 be applied.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Become a master of the escaping hell by inserting debug code into your scripts
 (reporting to 
\family typewriter
/dev/stderr
\family default
 or to log files) and do thorough testing like a devil.
\end_layout

\begin_layout Description

\family typewriter
@escvar{
\emph on
varname
\emph default
}
\family default
 Equivalent to 
\family typewriter
@esc{@{
\emph on
varname
\emph default
}}
\family default
.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

When creating a new resource via 
\family typewriter
marsadm create-resource
\family default
, or when adding a new replica via 
\family typewriter
marsadm join-resource
\family default
 or similar, the template system will automatically create new instances
 for the new resource or its replicas.
 Conversely, 
\family typewriter
marsadm leave-resource
\family default
 and its friends like 
\family typewriter
delete-resource
\family default
 etc will automatically remove the corresponding template instances from
 
\family typewriter
/run/systemd/system/
\family default
.
\end_layout

\begin_layout Subsection
Example 
\family typewriter
systemd
\family default
 Templates
\begin_inset CommandInset label
LatexCommand label
name "subsec:Example-systemd-Templates"

\end_inset


\end_layout

\begin_layout Standard
These can be found in the MARS repo in the 
\family typewriter
systemd/
\family default
 subdirectory.
 At the moment, the following are available (subject to further extension
 and improvements without notice):
\end_layout

\begin_layout Description

\family typewriter
mars.path
\family default
 This ensures that the mountpoint 
\family typewriter
/mars/
\family default
 is already mounted before 
\family typewriter
mars.service
\family default
 is started.
\end_layout

\begin_layout Description

\family typewriter
mars.service
\family default
 This starts and stops the MARS kernel module, provided that 
\family typewriter
/mars
\family default
 is (somehow) mounted.
 The latter can be ensured by classical 
\family typewriter
/etc/fstab
\family default
 methods, or by 
\family typewriter
.mount
\family default
 units like your own hand-crafted 
\family typewriter
mars.mount
\family default
 unit.
\end_layout

\begin_layout Description

\family typewriter
mars-trigger.path
\family default
 This is used for remote triggering of the marsadm template engine from
 another MARS cluster member, e.g.
 when initiating a handover.
 Local triggering is also possible via 
\family typewriter
touch /mars/userspace/systemd-trigger
\family default
.
 When triggered, the command 
\family typewriter
marsadm systemd-trigger
\family default
 (without 
\family typewriter
--force
\family default
) will be executed.
 In turn, this will re-compute all 
\family typewriter
systemd
\family default
 templates and start those units where the local host is in primary role.
\end_layout

\begin_layout Description

\family typewriter
dev-mars-@{res}.path
\family default
 This is used for generic triggering of any 
\family typewriter
systemd
\family default
 unit as set by 
\family typewriter
marsadm set-systemd-unit $res $unit
\family default
 (see below in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Handover-using-systemd"

\end_inset

).
\end_layout

\begin_layout Description

\family typewriter
^{mntname}-@{res}.mount
\family default
 This is one of the possible sub-ordinate targets which depend on 
\family typewriter
dev-mars-@{res}.path
\family default
.
 For fully automatic activation of this target, use something like 
\family typewriter
marsadm set-systemd-unit mydata vol-mydata.mount
\family default
 or similar.
 This will automatically mount 
\family typewriter
/dev/mars/mydata
\family default
 onto the mountpoint 
\family typewriter
/vol/mydata
\family default
.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Notice: the template notation 
\family typewriter
^{mntname}
\family default
 can be used for mounting to an 
\emph on
arbitrary
\emph default
 mountpoint, such as
\family typewriter
 /another/mountdir/mydata
\family default
, by using the corresponding systemd template syntax in 
\family typewriter
marsadm set-systemd-unit mydata another-mountdir-mydata.mount
\family default
.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 Look into the template file 
\family typewriter
^{mntname}-@{res}.mount
\family default
.
 In the first line, there is the following macro call:
\begin_inset Newline newline
\end_inset


\family typewriter
@eval{%let{mntpath}{%subst{%{mntname}}{-}{/}}}
\family default

\begin_inset Newline newline
\end_inset

This is a trick for conversion of any systemd template name 
\family typewriter
mntname
\family default
 into into an ordinary filesystem pathname 
\family typewriter
mntpath
\family default
.
 While subdirectories in a path are separated by slashes, the systemd unit
 naming conventions (as required by systemd) are using dashes in place of
 slashes.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Do not confuse 
\family typewriter
@{mntname}
\family default
 with 
\family typewriter
@{mntpath}
\family default
.
 Depending on the 
\emph on
type
\emph default
 of argument to be substituted, you may need either systemd unit naming
 conventions, or classical Unix pathname conventions.
\end_layout

\begin_layout Description

\family typewriter
^{mntname}-@{res}-testload.service
\family default
 This is an academic example for testing and for inspection, not intended
 for production.
 Here you can see how a 
\series bold
transitive dependency chain
\series default
 should be configured.
 In its body, this template contains a 
\family typewriter
BindsTo=
\family default
 reference to another template 
\family typewriter
^{mntname}-@{res}-medium-02.service
\family default
, which in turn contains a 
\family typewriter
BindsTo=
\family default
 reference to 
\family typewriter
^{mntname}-@{res}-medium-01.service
\family default
, which eventually references 
\family typewriter
^{mntname}-@{res}.mount
\family default
 also via 
\family typewriter
BindsTo=
\family default
.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Do not confuse 
\family typewriter
Requires=
\family default
 with 
\family typewriter
BindsTo=
\family default
 (see 
\family typewriter
man systemd.unit
\family default
).
 If you want to automatically stop your 
\emph on
entire
\emph default
 unit stack via a 
\emph on
single
\emph default
 command 
\family typewriter
systemctl stop vol-mydata.mount
\family default
, then you most likely need the stronger 
\family typewriter
BindsTo=
\family default
 directive in place of weaker ones like 
\family typewriter
Requires=
\family default
 or similar.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 In many cases (but not always), you also need an 
\family typewriter
After=
\family default
 directive.
 Otherwise you will unintentionally program a hard to debug 
\series bold
race condition
\series default
, which can extinct your last hair.
 Be sure to understand the corresponding details in the systemd documentation.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 In general, it is good practice to have a 
\emph on
consistent
\emph default
 name scheme.
 Always use the 
\emph on
logically same name
\emph default
 (modulo some escaping conventions for special characters), e.g.
 for the underlying LV (called disk in MARS terminology), equal to the MARS
 resource name, equal to the last part of the mountpoint, equal to the IQN
 of an iSCSI export, equal to the NFS share name, equal to the LXC container
 name, equal to the KVM/qemu virtual machine name, and so on.
 Messing around with non-systematic naming conventions can easily result
 in a hell.
\end_layout

\begin_layout Subsection
Handover involving 
\family typewriter
systemd
\begin_inset CommandInset label
LatexCommand label
name "subsec:Handover-using-systemd"

\end_inset


\end_layout

\begin_layout Standard
First, you need to install your systemd templates into one of the template
 directories mentioned in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Working-Principle-of"

\end_inset

.
 In case you have never used the template engine before, you can create
 the first instantiation via 
\family typewriter
marsadm systemd-trigger --force
\family default
.
 Afterwards, inspect 
\family typewriter
/run/systemd/system/
\family default
 for newly created template instances and check them.
\end_layout

\begin_layout Standard
For each resource 
\family typewriter
$res
\family default
, you should set (potentially different) systemd targets via 
\family typewriter
marsadm set-systemd-unit $res 
\begin_inset Quotes eld
\end_inset

$start_unit
\begin_inset Quotes erd
\end_inset

 
\begin_inset Quotes eld
\end_inset

$stop_unit
\family default

\begin_inset Quotes erd
\end_inset

.
 Notice that 
\family typewriter
$start_unit
\family default
 and 
\family typewriter
$stop_unit
\family default
 are typically denoting 
\emph on
different
\emph default
 targets for start and stop (with few exceptions) for the following reason:
\end_layout

\begin_layout Description
Example: assume your stack consists of 
\family typewriter
vol-@{res}.mount
\family default
 and 
\family typewriter
nfs-export-@{res}.service
\family default
.
 Before the filesystem can be exported via 
\family typewriter
nfs
\family default
, it 
\emph on
first
\emph default
 needs to be mounted.
 At startup, 
\family typewriter
systemd
\family default
 can do this easily for you: just add a 
\family typewriter
Requires=
\family default
 dependency between both targets, or similar.
 Then, simply use 
\family typewriter
nfs-export-mydata.service
\family default
 as start unit.
 Whenever it needs to be started, systemd will automatically analyze its
 dependencies and automatically start 
\family typewriter
vol-mydata.mount
\family default
.
 However, stopping is different.
 Theoretically, 
\family typewriter
systemctl stop nfs-export-mydata.service
\family default
 
\emph on
could
\emph default
 work in some cases, but in general it is not reliable.
 Reason: there might be other 
\emph on
sister
\emph default
 units which 
\emph on
also
\emph default
 depend on the mount.
 In some cases, you need not necessarily notice any sisters, because systemd
 can add further (internal) targets 
\emph on
automatically
\emph default
.
 The problem is easily solvable by 
\family typewriter
systemctl stop vol-mydata.mount
\family default
, which will automatically tear down 
\emph on
all
\emph default
 dependencies in reverse order.
 Therefore, use the 
\emph on
bottom
\emph default
 of the stack (usually a mount unit) as your stop unit.
\end_layout

\begin_layout Standard
For maximum safety, 
\family typewriter
$start_unit
\family default
 should always point at the 
\emph on
tip
\emph default
 of your stack, while 
\family typewriter
$stop_unit
\family default
 should always point at the 
\emph on
bottom
\emph default
 (but one level higher than 
\family typewriter
/dev/mars/$res
\family default
).
\end_layout

\begin_layout Standard
Removing any systemd targets is also possible via 
\family typewriter
marsadm set-systemd-unit $res 
\begin_inset Quotes eld
\end_inset


\begin_inset Quotes erd
\end_inset


\family default
 .
\end_layout

\begin_layout Standard
When everything is set up properly, the following should work:
\end_layout

\begin_layout Enumerate
Issue 
\family typewriter
marsadm primary $res
\family default
 on another node which is currently in secondary role.
\end_layout

\begin_layout Enumerate
As a consequence, 
\family typewriter
systemctl stop 
\begin_inset Quotes eld
\end_inset

$stop_unit
\begin_inset Quotes erd
\end_inset


\family default
 should be automatically executed at the old primary side.
\end_layout

\begin_layout Enumerate
After a while, the MARS kernel module will notice that 
\family typewriter
/dev/mars/$res
\family default
 is no longer opened.
 You can check this manually via
\family typewriter
 marsadm view-device-opened $res
\family default
 which will tell you a boolean result.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 In case the device is not closed for any reason, ordinary handover 
\emph on
cannot
\emph default
 proceed, because somebody could (at least potentially) write some data
 into it, even after the handover, which would lead to a split brain.
 Therefore MARS 
\emph on
must
\emph default
 insist that the device is closed before 
\emph on
ordinary
\emph default
 handover can proceed.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 In order to not leave you with a failed service, umount failures will be
 detected after a timeout.
 Handover by marsadm will then 
\emph on
automatically
\emph default
 restart the old start unit at the old primary side where the the device
 was not released.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 In case an ordinary handover is not possible due to hanging device openings,
 you have the following options:
\end_layout

\begin_deeper
\begin_layout Enumerate
Check your 
\family typewriter
systemd
\family default
 configuration or other sources of error why the device is not closed.
 Possible reasons could be hanging processes or hanging sessions which might
 need a 
\family typewriter
kill
\family default
 or a 
\family typewriter
kill -9
\family default
 or similar.
 Notice that 
\family typewriter
lsof
\family default
 does not catch 
\emph on
all
\emph default
 possible sources like (recursive or bind-) mounts occupied by foreign kernel
 namespaces.
\end_layout

\begin_layout Enumerate
Do a failover via 
\family typewriter
primary --force
\family default
, which will likely provoke a split brain.
\end_layout

\end_deeper
\begin_layout Enumerate
Once 
\family typewriter
/dev/mars/$res
\family default
 has disappeared, the ordinary MARS handover from the old primary to the
 new site should proceed as usual.
\end_layout

\begin_layout Enumerate
After 
\family typewriter
/dev/mars/$res
\family default
 has appeared at the new site, 
\family typewriter
systemctl start 
\begin_inset Quotes eld
\end_inset

$start_unit
\begin_inset Quotes erd
\end_inset


\family default
 should be automatically executed.
\end_layout

\begin_layout Standard
The rest depends on your 
\family typewriter
systemd
\family default
 and its configuration.
 For example, you can configure systemd targets for activation of VMs, or
 for 
\family typewriter
LXC
\family default
 containers, or for 
\family typewriter
iSCSI
\family default
 exports, or for 
\family typewriter
nfs
\family default
 exports, or for 
\family typewriter
glusterfs
\family default
 exports, or for whatever you need.
 For true geo-redundancy, you will likely have to include some 
\family typewriter
quagga
\family default
 or 
\family typewriter
bird
\family default
 or other BGP configurations into your systemd unit stack.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 In general, marsadm tries to keep your services running whenever a handover
 failure occurs, or when you re-attach after a detach, or when your machine
 reloads 
\family typewriter
mars.ko
\family default
 after a crash reboot, etc.
 This is regarded as a 
\emph on
feature
\emph default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 However, this feature could become boring if you 
\emph on
intentionally(!)
\emph default
 want to stop your services, for example when you need to run an 
\family typewriter
fsck
\family default
.
 Do not use 
\family typewriter
marsadm secondary
\family default
, because this would make 
\family typewriter
/dev/mars/mydata
\family default
 to disappear.
 Although 
\family typewriter
marsadm set-systemd-unit mydata 
\begin_inset Quotes eld
\end_inset


\begin_inset Quotes erd
\end_inset


\family default
 would solve the problem, this would make you forget the old start and stop
 unit names.
 You could workaround by some wrapper script remembering the old names via
 
\family typewriter
marsadm get-systemd-unit
\family default
, but this is not necessary:
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

 There is a simple solution: 
\family typewriter
marsadm set-systemd-want 
\begin_inset Quotes eld
\end_inset

(none)
\begin_inset Quotes erd
\end_inset


\family default
 will 
\emph on
temporarily
\emph default
 stop the whole systemd unit stack, while keeping 
\family typewriter
/dev/mars/mydata
\family default
 accessible.
 After your 
\family typewriter
fsck /dev/mars/mydata
\family default
 has finished, simply use an idempotent 
\family typewriter
marsadm primary mydata
\family default
 for restart of your services.
\end_layout

\begin_layout Section
The macro processor
\begin_inset CommandInset label
LatexCommand label
name "sec:The-macro-processor"

\end_inset


\end_layout

\begin_layout Standard
The macro processor is a very flexible and versatile tool for 
\series bold
customizing
\series default
.
 Conceptually, two levels of macros are discriminated:
\end_layout

\begin_layout Enumerate
primitive macros: these are firmly built into 
\family typewriter
marsadm
\family default
.
\end_layout

\begin_layout Enumerate
complex macros: these can be defined via the 
\series bold
macro language
\series default
 of 
\family typewriter
marsadm
\family default
.
\end_layout

\begin_layout Standard
Some complex macros are already pre-defined, for example the standard 
\family typewriter
marsadm view all
\family default
 (see section 
\begin_inset CommandInset ref
LatexCommand vref
reference "subsec:Standard-marsadm-view"
plural "false"
caps "false"
noprefix "false"

\end_inset

).
\end_layout

\begin_layout Standard
From the commandline, any macro can be called via 
\family typewriter
marsadm view-
\emph on
$macroname
\emph default
 mydata
\family default
.
 The short form 
\family typewriter
marsadm view mydata
\family default
 is equivalent to 
\family typewriter
marsadm view-default mydata
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

In general, the command 
\family typewriter
marsadm view-
\emph on
$macroname
\emph default
 all
\family default
 will first call the macro 
\family typewriter
\emph on
$macroname
\family default
\emph default
 in a loop for 
\emph on
all
\emph default
 resources we are a 
\emph on
member locally
\emph default
.
 Finally, a trailing macro 
\family typewriter
\emph on
$macroname
\emph default
-global
\family default
 will be called with an empty 
\family typewriter
%{res}
\family default
 argument, provided that such a macro is defined.
 This way, you can produce per-resource output followed by global output
 which does not depend on a particular resource.
\end_layout

\begin_layout Subsection
Predefined Primitive Macros
\begin_inset CommandInset label
LatexCommand label
name "subsec:Predefined-Trivial-Macros"

\end_inset


\end_layout

\begin_layout Subsubsection
Intended for Humans
\end_layout

\begin_layout Standard
In the following, shell glob notation 
\family typewriter
{a,b}
\family default
 is used to document similar variants of similar macros in a single place.
 When you actually call the macro, you must choose one of the possible variants
 (excluding the braces).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
the-err-msg
\family default
 Show reported errors for a resource.
 When the resource argument is missing or empty, show global error information.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
all-err-msg
\family default
 Like before, but show all information including those which are 
\family typewriter
OK
\family default
.
 This way, you get a list
\begin_inset Foot
status open

\begin_layout Plain Layout
The list may be extended in future versions of MARS.
\end_layout

\end_inset

 of 
\emph on
all
\emph default
 potential error information present in the system.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-wrn-msg
\family default
 Show all / reported warnings in the system.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-inf-msg
\family default
 Show all / reported informational messages in the system.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-msg
\family default
 Show all / reported messages regardless of its classification.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-global-msg
\family default
 Show global messages not associated with any resource (the resource argument
 of the 
\family typewriter
marsadm
\family default
 command is ignored in this case).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-global-{inf,wrn,err}-msg
\family default
 Dito, but more specific.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-pretty-{global-,}{inf-,wrn-,err-,}msg
\family default
 Dito, but show numerical timestamps in a human readable form.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{all,the}-{global-,}{inf-,wrn-,err-,}count
\family default
 Instead of showing the messages, show their count (number of lines).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
errno-text
\family default
 This macro takes 1 argument, which must represent a Linux 
\family typewriter
errno
\family default
 number, and converts it to human readable form (similar to the C 
\family typewriter
strerror()
\family default
 function).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
todo-{attach,sync,fetch,replay,primary,secondary}
\family default
 Shows a boolean value (0 or 1) indicating the current state of the correspondin
g todo switch (whether on or off).
 The meaning of todo switches is illustrated in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-State-of"

\end_inset

.
 Exceptions: 
\family typewriter
todo-primary
\family default
 is not reporting the boolean value of a switch, but means that the designated
 primary string as reported by 
\family typewriter
get-primary
\family default
 is equal to the current host.
 Similary, 
\family typewriter
todo-secondary
\family default
 means that no designated primary exists throughout the cluster, indicating
 that 
\family typewriter
get-primary
\family default
 equals to the special string value 
\family typewriter
(none)
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
get-resource-{fat,err,wrn}
\family default
 Access to the internal error status files.
 This is not an official interface and may thus change at any time without
 notice.
 Use this only for human inspection, not for scripting!
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 These macros, as well as the error status files, are likely to disappear
 in future versions of MARS.
 They should be used for debugging only.
 At least when merging into the upstream Linux kernel, only the 
\family typewriter
*-msg
\family default
 macros will likely survive.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
get-resource-{fat,err,wrn}-count
\family default
 Dito, but get the number of lines instead of the text.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
replay-code
\family default
 Indicate the current state of logfile replay / recovery:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000
(empty) Unknown.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
0 No replay is currently running.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
1 Replay is currently running.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
2 Replay has successfully stopped.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
<0 See Linux 
\family typewriter
errno
\family default
 code.
 Typically this indicates a damaged logfile, or another filesystem error
 at 
\family typewriter
/mars
\family default
.
\end_layout

\end_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-{attach,sync,fetch,replay,primary,secondary,module-loaded}
\family default
 Shows a boolean value (0 or 1) indicating the 
\emph on
actual
\emph default
 state, whether the corresponding action has been actually carried out,
 or not (yet).
 Notice that the values indicated by 
\family typewriter
is-*
\family default
 may differ from the 
\family typewriter
todo-*
\family default
 values when something is not (yet) working.
 Notice: 
\family typewriter
is-primary
\family default
 (or its negation 
\family typewriter
is-secondary
\family default
) means that the transaction logger has (resp.
 not) reached a working state, but the corresponding 
\family typewriter
/dev/mars/mydata
\family default
 prosumer device need not (yet) have appeared (somewhere else).
 More explanations can be found in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-State-of"

\end_inset

.
 
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-split-brain
\family default
 Shows whether split brain (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Split-Brain-Resolution"

\end_inset

) has been detected, or not.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-consistent
\family default
 Shows whether the 
\emph on
underlying disk
\emph default
 is in a locally consistent state, i.e.
 whether it 
\emph on
could
\emph default
 be (potentially) detached and then used for read-only test-mounting
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that the 
\emph on
writeback
\emph default
 at the primary side is out-of-order by default, for performance reasons.
 Therefore, the underlying disk is only guaranteed to be consistent when
 there is no data left to be written back.
 Notice that this condition is racy by construction.
 When your primary node crashes during writeback and then comes up again,
 you must do a 
\family typewriter
modprobe mars
\family default
 first in order to automatically replay the transaction logfiles, which
 will automatically heal such temporary inconsistencies.
\end_layout

\end_inset

.
 Don't confuse this with the consistency of 
\family typewriter
/dev/mars/mydata
\family default
, which is by construction 
\emph on
always
\emph default
 locally consistent once it has appeared
\begin_inset Foot
status open

\begin_layout Plain Layout
Exceptions are possible when using 
\family typewriter
marsadm fake-sync
\family default
.
 Even in split brain situations, 
\family typewriter
marsadm primary --force
\family default
 tries to prevent any further potential exception as best as it can, by
 not letting 
\family typewriter
/dev/mars/mydata
\family default
 to appear and by insisting on split brain resolution first.
 In future implementations, this might change if more pressure is put on
 the developer to sacrifice consistency in preference to not waiting for
 a full logfile replay.
\end_layout

\end_inset

.
 By construction of MARS, the disk of secondaries will 
\emph on
always
\emph default
 remain in a locally consistent state once the initial sync has finished
 as well as the initial logfile replay.
 Notice that local consistency does not necessarily imply actuality.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-emergency
\family default
 Shows whether emergency mode (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Emergency-Mode"

\end_inset

) has been entered for the named resource, or not.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
nr-{attach,sync,fetch,replay,primary,secondary}
\family default
 Show the 
\emph on
total number
\emph default
 of resource members which are in corresponding state 
\family typewriter
%is-
\emph on
$something
\emph default
{}
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
rest-space
\family default
 (global, no resource argument necessary) Shows the 
\emph on
logically
\emph default
 available space in 
\family typewriter
/mars/
\family default
, which may deviate from the physically available space as indicated by
 the 
\family typewriter
df
\family default
 command.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
get-{disk,device}
\family default
 Show the name of the underlying disk, or of the 
\family typewriter
/dev/mars/mydata
\family default
 device (if it is available).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{disk,device}-present
\family default
 Show (as a boolean value) whether the underlying disk, or the 
\family typewriter
/dev/mars/mydata
\family default
 device, is available.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-opened
\family default
 Show (as a number) how often 
\family typewriter
/dev/mars/mydata
\family default
 has been actually openend, e.g.
 by 
\family typewriter
mount
\family default
 or by some processes like 
\family typewriter
dd
\family default
, or by iSCSI, etc.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-{ops,amount}-rate
\family default
 Show the number of current IOPS, esp.
 the current throughput in KiB/s.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-nrflying
\family default
 Show the number of currently flying IO requests.
 This is an indicator of queueing at the low-level device.
 When it is permenantly very high, it may point at IO problems, such as
 RAID degradation.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
disk-error
\family default
 Show the negative Linux errno code of the last open() error on the underlying
 disk.
 It should be always zero.
 When < 0 according to kernel return-code conventions, this typically indicates
 a hardware or LVM problem, etc.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-error
\family default
 Show the negative Linux errno code of the last IO error, as reported upwards
 to applications.
 It should be always zero.
 When < 0 according to kernel return-code conventions, this typically indicates
 a hardware (or network) problem.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
{potential,implemented,usable}-features Show a list of flag names, indicating
 the compression / digest features (see description in section 
\begin_inset CommandInset ref
LatexCommand formatted
reference "sec:Data-Compression-Checksumming"
plural "false"
caps "false"
noprefix "false"

\end_inset

) as either as known to the current version of marsadm, or as implemented
 in the currently running kernel module, or as the minimum feature set currently
 available in the whole cluster.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
{implemented,usable}-{digests,compressions} Same as before, but more specificall
y related to either compressions or digests.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
enabled-{log|net}-compressions Show which compression features have been
 set by 
\family typewriter
marsadm set-global-enabled-*-compressions
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
disabled-{log|net}-digests Show which digest features have been disabled
 by 
\family typewriter
marsadm set-global-disabled-*-digests
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000
used-{log,net}-{digest,compression} Show which digest or compression features
 are currently actually used by $host, either for logfile or for network
 purposes.
\end_layout

\begin_layout Subsubsection
Intended for Scripting
\end_layout

\begin_layout Standard
While complex macros may output a whole bunch of information, the following
 primitive macros are outputting exactly one value.
 They are intended for script usage (cf.
 section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Scripting-HOWTO"

\end_inset

).
 Of course, curious humans may also use them :)
\end_layout

\begin_layout Standard
In the following, shell glob notation 
\family typewriter
{a,b}
\family default
 is used to document similar variants of similar macros in a single place.
 When you actually call the macro, you must choose one of the possible variants
 (excluding the braces).
\end_layout

\begin_layout Paragraph
Memberships, Name Querying and their Counts
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-member
\family default
 Boolean, indicating whether 
\family typewriter
%{host}
\family default
 is a storage member of the resource 
\family typewriter
%{res}.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-guest
\family default
 Boolean, indicating whether 
\family typewriter
%{host}
\family default
 is currently a 
\emph on
dynamic guest
\emph default
 of resource 
\family typewriter
%{res}.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
cluster-peers
\family default
 Show a newline-separated list of all host names participating in the cluster.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
resource-peers
\family default
 Show a newline-separated list of all host names participating in the particular
 resource 
\family typewriter
%{res}
\family default
.
 Notice that this is typically a subset of 
\family typewriter
%cluster-peers{}
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
guest-peers
\family default
 Show a newline-separated list of all host names which are currently dynamically
 added as 
\emph on
guests
\emph default
 to resource 
\family typewriter
%{res}
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
count-{cluster,resource,guest}-peers
\family default
 Show the corresponding 
\emph on
number
\emph default
 of hosts, accordingly.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{my,all}-resources
\family default
 Show a newline-separated list of either all resource names 
\emph on
existing
\emph default
 in the cluster, or only those where the current host 
\family typewriter
%{host}
\family default
 is a storage member.
 Optionally, you may specify the hostname as a parameter, e.g.
 
\family typewriter
%my-resources{
\emph on
otherhost
\emph default
}
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{my,all}-members
\family default
 Show a newline-separated list of storage members existing in the cluster.
 There is a very subtle difference to 
\family typewriter
*-resources
\family default
: there may exist resources which have no storage members.
 This may for example occur when all storage members have left via leave-resourc
e, but delete-resource has not yet been executed.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{my,all}-guests
\family default
 Show a newline-separated list of currently dynamically added guests.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
count-{my,all}-{resources,members,guests}
\family default
 Show the corresponding 
\emph on
number
\emph default
 of resources or storage members or guests, accordingly.
\end_layout

\begin_layout Paragraph
Amounts of Data Inquiry
\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\noindent
\align center
\begin_inset Graphics
	filename images/fetch-replay-total.fig
	width 80col%

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption Standard

\begin_layout Plain Layout
overview on amounts / cursors
\begin_inset CommandInset label
LatexCommand label
name "fig:overview-on-amounts"

\end_inset


\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
\noindent
The following macros are meaningful for both primary and secondary nodes:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
deletable-size
\family default
 Show the total amount of 
\emph on
locally present
\emph default
 logfile data which 
\emph on
could
\emph default
 be deleted by 
\family typewriter
marsadm log-delete-all mydata
\family default
.
 This differs almost always from both 
\family typewriter
replay-pos
\family default
 and 
\family typewriter
occupied-size
\family default
 due to granularity reasons (only whole logfiles can be deleted).
 Units are 
\emph on
bytes
\emph default
, not kilobytes.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
occupied-size
\family default
 Show the total amount of 
\emph on
locally present
\emph default
 logfile data (sum of all file sizes).
 This is often roughly approximate to 
\family typewriter
fetch-pos
\family default
, but it may differ vastly (in both directions) when logfiles are not completely
 transferred, when some are damaged, during split brain, after a 
\family typewriter
join-resource
\family default
 / 
\family typewriter
invalidate
\family default
, or when the resource is in emergency mode (see section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Emergency-Mode"

\end_inset

).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
disk-size
\family default
 Show the size of the underlying local disk in bytes.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
resource-size
\family default
 Show the logical size of the resource in bytes.
 When this value is lower than 
\family typewriter
disk-size
\family default
, you are wasting space.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-size
\family default
 At a primary node, this may differ from 
\family typewriter
resource-size
\family default
 only for a very short time during the 
\family typewriter
resize
\family default
 operation.
 At secondaries, there will be no difference.
\end_layout

\begin_layout Standard
\noindent
The following macros are only meaningful for resources in primary mode:
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
writeback-rest
\family default
 Show the amount of data which is already in the transaction logfile, but
 has not yet been written back to the underlying disk.
 This may be used for estimation of recovery time after a potential primary
 crash.
 The writeback buffer is explained by the graphics at 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:The-Transaction-Logger"
plural "false"
caps "false"
noprefix "false"

\end_inset

.
\end_layout

\begin_layout Standard
\noindent
The following macros are only meaningful for resources in secondary mode.
 By information theoretic limits, they can only tell what is 
\emph on
locally known
\emph default
.
 They 
\series bold
cannot
\series default
 reflect the 
\begin_inset Quotes eld
\end_inset

true (global) state
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that according to Einstein's law, and according to observations by
 Lamport, the concept of 
\begin_inset Quotes eld
\end_inset

true state
\begin_inset Quotes erd
\end_inset

 does not exist at all in a distributed system.
 Anything you can know in a distributed system is always local knowlege,
 which races with other (remote) knowlege, and may be outdated at 
\emph on
any
\emph default
 time.
\end_layout

\end_inset


\begin_inset Quotes erd
\end_inset

 of a cluster, in particular during network partitions.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-size
\family default
 Show the total amount of data which is / was to be processed by either
 sync, fetch, or replay.
 
\family typewriter
work-size
\family default
 is equivalent to 
\family typewriter
fetch-size
\family default
.
 
\family typewriter
replay-size
\family default
 is equivalent to 
\family typewriter
fetch-pos
\family default
 (see below).
 Units are 
\emph on
bytes
\emph default
, not kilobytes.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-pos
\family default
 Show the total amount of data which is already processed (current 
\begin_inset Quotes eld
\end_inset

cursor
\begin_inset Quotes erd
\end_inset

 position).
 
\family typewriter
work-pos
\family default
 is equivalent to 
\family typewriter
replay-pos
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

The 0% point is the 
\emph on
locally contiguous
\emph default
 amount of data since the last 
\family typewriter
create-resource
\family default
, 
\family typewriter
join-resource
\family default
, or 
\family typewriter
invalidate
\family default
, or since the last emergency mode, but possibly shortened by 
\family typewriter
log-delete
\family default
s.
 Notice that the 0% point may be different on different cluster nodes, because
 their resource history may be different or non-contiguous during split
 brain, or after a 
\family typewriter
join-resource
\family default
, or after 
\family typewriter
invalidate
\family default
, or during / after emergency mode.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-rest
\family default
 Shows the difference between 
\family typewriter
*-size
\family default
 and 
\family typewriter
*-pos
\family default
 (amount of work to do).
 
\family typewriter
work-rest
\family default
 is therefore the difference between 
\family typewriter
fetch-size
\family default
 and 
\family typewriter
replay-pos
\family default
, which is the 
\emph on
total
\emph default
 amount of work to do (regardless whether to be fetched and/or to be replayed).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-reached
\family default
 Boolean value indicating whether 
\family typewriter
*-rest
\family default
 dropped down to zero
\begin_inset Foot
status open

\begin_layout Plain Layout
MARS can only guarantee local consistency, but cannot guarantee actuality
 in all imaginable situations.
 Notice that a general notion of 
\begin_inset Quotes eld
\end_inset

actuality
\begin_inset Quotes erd
\end_inset

 is 
\emph on
undefinable
\emph default
 in a widely distributed system at all, according to Einstein's laws.
\end_layout

\begin_layout Plain Layout
Let's look at an example.
 In case of a node crash, and after the node is up again, a 
\family typewriter
modprobe mars
\family default
 has to occur, in order to replay the transaction logs of MARS again.
 However, at the recovery phase before, the journalling 
\family typewriter
ext4
\family default
 filesystem 
\family typewriter
/mars/
\family default
 
\emph on
may
\emph default
 have rolled back some internal symlink updates which have occurred immediately
 before the crash.
 MARS is relying on the fact that journalling filesystems like 
\family typewriter
ext4
\family default
 should do their recovery in a consistent way, possibly by sacrifycing actuality
 a little bit.
 Therefore, the above macros cannot guarantee to deliver true information
 about what is persisted at the moment.
\end_layout

\begin_layout Plain Layout
Notice that there are further potential caveats.
\end_layout

\begin_layout Plain Layout
In case of 
\family typewriter
{sync,fetch}-reached
\family default
, MARS uses 
\family typewriter
bio
\family default
 callbacks resp.
 
\family typewriter
fdatasync()
\family default
 by default, thus the underlying storage layer has 
\emph on
told
\emph default
 us that it 
\emph on
believes
\emph default
 it has commited the data in a reboot-safe way.
 Whether this is 
\emph on
really
\emph default
 true does not depend on MARS, but on the lower layers of the storage hierarchy.
 There exists hardware where this claim is known to be wrong under certain
 circumstances, such as certain hard disk drives in certain modes of operation.
 Please check the hardware for any violations of storage semantics under
 certain circumstances such as power loss, and check information sources
 like magazines about the problem area.
 Please notice that such a problem, if it exists at all, is independent
 from MARS.
 It would also exist if you wouldn't use MARS on the same system.
\end_layout

\end_inset

.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{fetch,replay,work}-threshold-reached
\family default
 Boolean value indicating whether 
\family typewriter
*-rest
\family default
 dropped down to 
\family typewriter
%{threshold}
\family default
, which is pre-settable by the 
\family typewriter
--threshold=
\emph on
size
\family default
\emph default
 command line option (default is 10 MiB).
 In asynchronous use cases of MARS, this should be preferred over 
\family typewriter
*-reached
\family default
 for 
\emph on
human display
\emph default
, because it produces less flickering by the inevitable replication delay.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{fetch,replay,work}-almost-reached
\family default
 Boolean value indicating whether 
\family typewriter
*-rest
\family default
 
\emph on
almost
\emph default
 / 
\emph on
approximately
\emph default
 dropped down to zero.
 The default is that at lease 990 permille are reached.
 In asynchronous use cases of MARS, this can be preferred over 
\family typewriter
*-reached
\family default
 for 
\emph on
human display
\emph default
 only, because it produces less flickering by the inevitable replication
 delay.
 However, don't base any decisions on this!
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-percent
\family default
 The cursor position 
\family typewriter
*-pos
\family default
 as a percentage of 
\family typewriter
*-size
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-permille
\family default
 The cursor position 
\family typewriter
*-pos
\family default
 as permille of 
\family typewriter
*-size
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-rate
\family default
 Show the current throughput in bytes
\begin_inset Foot
status open

\begin_layout Plain Layout
Notice that the internal granularity reported by the kernel may be coarser,
 such as KiB.
 This interfaces abstracts away from kernel internals and thus presents
 everything in byte units.
\end_layout

\end_inset

 per second.
 
\family typewriter
work-rate
\family default
 is the 
\emph on
maximum
\emph default
 of 
\family typewriter
fetch-rate
\family default
 and 
\family typewriter
replay-rate
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay,work}-remain
\family default
 Show the 
\emph on
estimated
\emph default
 remaining time for completion of the respective operation.
 This is just a very raw guess.
 Units are seconds.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{sync,fetch,replay}-{ops,amount}-rate
\family default
 Show the current IOPS of sync / fetch / replay, or its corresponding throughput
 in KiB/s.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
summary-vector
\family default
 Show the colon-separated CSV value 
\family typewriter
%replay-pos{}:%fetch-pos{}:%fetch-size{}
\family default
.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
replay-basenr 
\family default
Get currently first reachable logfile number (see figure 
\begin_inset CommandInset ref
LatexCommand vref
reference "fig:overview-on-amounts"

\end_inset

).
 Only for curious humans or for debugging / monitoring - don't base any
 decisions on this.
 Use the 
\family typewriter
*-{pos,size}
\family default
 macros instead.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{replay,fetch,work}-lognr 
\family default
Get current logfile number of replay or fetch position, or of the currently
 known last reachable number (see figure 
\begin_inset CommandInset ref
LatexCommand vref
reference "fig:overview-on-amounts"

\end_inset

).
 Only for curious humans or for debugging / monitoring - don't base any
 decisions on this.
 Use the 
\family typewriter
*-{pos,size}
\family default
 macros instead.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{replay,fetch,work}-logcount 
\family default
Get current number of logfiles which are already replayed, or are already
 fetched, or are to be applied in total (see figure 
\begin_inset CommandInset ref
LatexCommand vref
reference "fig:overview-on-amounts"

\end_inset

).
 Only for curious humans or for debugging / monitoring - don't base any
 decisions on this.
 Use the 
\family typewriter
*-{rest}
\family default
 macros instead.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
alive-timestamp
\family default
 Tell the Lamport Unix timestamp (seconds since 1970) of the last metadata
 communication to the designated primary (or to any other host given by
 the first argument).
 Returns 
\begin_inset Formula $-1$
\end_inset

 if no such host exists.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{fetch,replay,work}-timestamp
\family default
 Tell the Lamport Unix timestamp (seconds since 1970) when the last progress
 has been made.
 When no such action exists, 
\begin_inset Formula $-1$
\end_inset

 is returned.
 
\family typewriter
%work-timestamp{
\emph on
hostname
\emph default
}
\family default
 is the maximum of 
\family typewriter
%fetch-timestamp{
\emph on
hostname
\emph default
}
\family default
 and 
\family typewriter
%replay-timestamp{
\emph on
hostname
\emph default
}
\family default
.
 When the parameter 
\family typewriter
\emph on
hostname
\family default
\emph default
 is empty, the local host will be reported (default).
 Example usage: 
\family typewriter
marsadm view all --macro=
\begin_inset Quotes erd
\end_inset

%replay-timestamp{%todo-primary{}}
\begin_inset Quotes erd
\end_inset


\family default
 shows the timestamp of the last reported
\begin_inset Foot
status open

\begin_layout Plain Layout
Updates of this information are occurring with lower frequency than actual
 writebacks, for performance reasons.
 The metadata network update protocol will add further delays.
 Therefore, the accuracy is only in the range of minutes.
\end_layout

\end_inset

 writeback action at the designated primary.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{alive,fetch,replay,work}-age
\family default
 Tell the number of seconds since the last respective action, or 
\begin_inset Formula $-1$
\end_inset

 if none exists.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
{alive,fetch,replay,work}-lag
\family default
 Report the time difference (in seconds) between the last 
\emph on
known
\emph default
 action at the local host and at the designated primary (or between any
 other hosts when 2 parameters are given).
 Returns 
\begin_inset Formula $-1$
\end_inset

 if no such action exists at any of the two hosts.
 Attention! This need not reflect the 
\emph on
actual
\emph default
 state in case of networking problems.
 Don't draw wrong conclusions from a high 
\family typewriter
{fetch,replay}-lag
\family default
 value: it could also mean that simply no write operation at all has occurred
 at the primary side for a long time.
 Conversely, a low lag value does not imply that the replication is recent:
 it may refer to 
\emph on
different
\emph default
 write operations at each of the hosts; therefore it only tells that 
\emph on
some
\emph default
 progress has been made, but says nothing about the amount of the progress.
\end_layout

\begin_layout Paragraph
Device Information
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
get-device
\family default
 Tell the device name, which is 
\family typewriter
/dev/mars/%{res}
\family default
 in the current MARS implementation.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-present
\family default
 Boolean, telling whether 
\family typewriter
/dev/mars/%{res}
\family default
 is currently appearing at 
\family typewriter
%{host}
\family default
 or not.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-opened
\family default
 Tell the number of times 
\family typewriter
/dev/mars/%{res}
\family default
 is currently opened (e.g.
 mounted) at 
\family typewriter
%{host}
\family default
.
 Upon non-exclusive access by multiple readers / writers in parallel (which
 is potentially very dangerous), the number may grow greater than 1.
 You may exploit this for monitoring / supervision.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-ops-rate
\family default
 Tell the current request throughput, aka IOPS.
 This is actually changing much more frequently than can be reported by
 the kernel, but anyway may be useful for getting some impression on what
 is going on.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-error
\family default
 Tell the Unix error code when any IO error has occurred in the past, or
 0 when no error is known.
 Useful for debugging and fault analysis.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-nrflying
\family default
 Tell the number of currently flying IO requests (i.e.
 submitted, but not yet completed).
 This is changing in much higher frequency that can be ever reported by
 the kernel, but may be useful for bottleneck analysis, and when the system
 is stuck (e.g.
 defective RAID).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-completion-stamp
\family default
 Tell the realtime timestamp of the last completed IO request.
 Useful for detection of a hanging system (e.g.
 defective disks, etc).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
device-completion-age
\family default
 Similar to before, but report the 
\emph on
relative
\emph default
 age (compared to the current time) in seconds.
\end_layout

\begin_layout Paragraph
Misc Informational Status
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
get-primary
\family default
 Return the name of the current designated primary node as locally known.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
actual-primary
\family default
 (deprecated) try to determine the name of the node which 
\emph on
appears
\emph default
 to be the actual primary.
 This only a 
\series bold
\emph on
guess
\series default
\emph default
, because it is not generally unique in split brain situations! Don't use
 this macro.
 Instead, use 
\family typewriter
is-primary
\family default
 on those nodes you are interested in.
 The explanations from section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-State-of"

\end_inset

 also apply to 
\family typewriter
get-primary
\family default
 versus 
\family typewriter
actual-primary
\family default
 analogously.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
is-alive
\family default
 Boolean value indicating whether all other nodes participating in 
\family typewriter
mydata
\family default
 are reachable / healthy.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
uuid
\family default
 (global) Show the unique identifier created by 
\family typewriter
create-cluster
\family default
 or by 
\family typewriter
create-uuid
\family default
.
 Hint: this is immutable, and it is firmly bound to the 
\family typewriter
/mars/
\family default
 filesystem.
 It can only be destroyed by deleting the whole filesystem (see section
 
\begin_inset CommandInset ref
LatexCommand ref
reference "leave-cluster"

\end_inset

).
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
tree
\family default
 (global) Indicate symlink tree version (see 
\family typewriter
mars-for-kernel-developers.pdf
\family default
).
\end_layout

\begin_layout Paragraph
Experts Only
\end_layout

\begin_layout Standard
The following is for hackers who know what they are doing.
 The following is not officially supported.
\end_layout

\begin_layout Labeling
\labelwidthstring 00.00.0000

\family typewriter
wait-{is,todo}-{attach,sync,fetch,replay,primary,secondary}-{on,off}
\family default
 This may be used to program some useful waiting conditions in advanced
 macro scripts.
 It works via busy wait, and does not support disjoint waiting conditions.
 Use at your own risk! Hint: for disjoint and/or more complex waiting conditions
, and/or for programming your own finite state transition machines etc,
 please prefer the non-blocking 
\family typewriter
{is,todo}-*
\family default
 and sisters, and program any busy wait yourself (or try to avoid busy-wait
 at all).
\end_layout

\begin_layout Section
Creating your own Macros
\begin_inset CommandInset label
LatexCommand label
name "subsec:Creating-your-own"

\end_inset


\end_layout

\begin_layout Standard
In order to create your own macros, you could start writing them from scratch
 with your favorite ASCII text editor.
 However, it is much easier to take an existing macro and to customize it
 to your needs.
 In addition, you can learn something about macro programming by looking
 at the existing macro code.
\end_layout

\begin_layout Standard
Go to a new empty directory and say
\end_layout

\begin_layout Itemize

\family typewriter
marsadm dump-macros
\end_layout

\begin_layout Standard
in order to get the most interesting complex macros, or say
\end_layout

\begin_layout Itemize

\family typewriter
marsadm dump-all-macros
\end_layout

\begin_layout Standard
in order to additionally get some primitive macros which could be customized
 if needed.
 This will write lots of files 
\family typewriter
*.tpl
\family default
 into your current working directory.
\end_layout

\begin_layout Standard
Any modfied or new macro file should be placed either into the current working
 directory 
\family typewriter
./
\family default
 , or into 
\family typewriter
$HOME/.marsadm/
\family default
 , or into 
\family typewriter
/etc/marsadm/
\family default
 .
 They will be searched in this order, and the first match will win.
 When no macro file is found, the built-in version will be used if it exists.
 This way, you may override builtin macros.
\end_layout

\begin_layout Standard
Example: if you have a file 
\family typewriter
./mymacro.tpl
\family default
 you just need to say 
\family typewriter
marsadm view-mymacro mydata
\family default
 in order to invoke it in the resource context 
\family typewriter
mydata
\family default
.
\end_layout

\begin_layout Subsection
General Macro Syntax
\end_layout

\begin_layout Standard
Macros are simple ASCII text, enriched with calls to other macros.
\end_layout

\begin_layout Standard
ASCII text outside of comments are copied to the output verbatim.
 Comments are skipped.
 Comments may have one of the following well-known forms:
\end_layout

\begin_layout Itemize

\family typewriter
# skipped text until / including next newline character
\end_layout

\begin_layout Itemize

\family typewriter
// skipped text until / including next newline character
\end_layout

\begin_layout Itemize

\family typewriter
/* skipped text including any newline characters */
\end_layout

\begin_layout Itemize
denoted as Perl regex: 
\family typewriter

\backslash

\backslash

\backslash
n
\backslash
s* 
\family default
(single backslash directly followed by a newline character, and eating up
 any whitespace characters at the beginning of the next line) Hint: this
 may be fruitfully used to structure macros in a more readable form / indentatio
n.
\end_layout

\begin_layout Standard
Special characters are always initiated by a backslash.
 The following pre-defined special character sequences are recognized:
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
n
\family default
 newline
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
r
\family default
 return (useful for DOS compatibility)
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
t
\family default
 tab
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
f
\family default
 formfeed
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
b
\family default
 backspace
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
a
\family default
 alarm (bell)
\end_layout

\begin_layout Itemize

\family typewriter

\backslash
e
\family default
 escape (e.g.
 for generating ANSI escape sequences)
\end_layout

\begin_layout Itemize

\family typewriter

\backslash

\family default
 followed by anything else: assure that the next character is taken verbatim.
 Although possible, please don't use this for escaping letters, because
 further escape sequences might be pre-defined in future.
 Best practice is to use this only for escaping the backslash itself, or
 for escaping the percent sign when you don't want to call a macro (protect
 against evaluation), or to escape a brace directly after a macro call (verbatim
 brace not to be interpreted as a macro parameter).
\end_layout

\begin_layout Itemize
All other characters stand for their own.
 If you like, you should be able to produce XML, HTML, JSON and other ASCII-base
d output formats this way.
\end_layout

\begin_layout Standard
Macro calls have the following syntax:
\end_layout

\begin_layout Itemize

\family typewriter
%
\emph on
macroname
\emph default
{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
}{
\emph on
argn
\emph default
}
\end_layout

\begin_layout Itemize
Of course, arguments may be empty, denoted as 
\family typewriter
{}
\end_layout

\begin_layout Itemize
It is possible to supply more arguments than required.
 These are simply ignored.
\end_layout

\begin_layout Itemize
There must be always at least 1 argument, even for parameterless macros.
 In such a case, it is good style to leave it empty (even if it is actually
 ignored).
 Just write 
\family typewriter
%parameterlessmacro{}
\family default
 in such a case.
\end_layout

\begin_layout Itemize

\family typewriter
%{
\emph on
varname
\emph default
}
\family default
 syntax: As a special case, the macro name may be empty, but then the first
 argument must denote a previously defined variable (such as assigned via
 
\family typewriter
%let{varname}{myvalue}
\family default
, or a pre-defined standard variable like 
\family typewriter
%{res}
\family default
 for the current resource name, see later paragraph 
\begin_inset CommandInset ref
LatexCommand ref
reference "par:Predefined-Variables"

\end_inset

).
\end_layout

\begin_layout Itemize
Of course, parameter calls may be (almost) arbitrarily nested.
\end_layout

\begin_layout Itemize
Of course, the 
\emph on
correctness
\emph default
 of nesting of braces must be generally obeyed, as usual in any other macro
 processor language.
 General rule: for each opening brace, there must be exactly one closing
 brace somewhere afterwards.
\end_layout

\begin_layout Standard
These rules are hopefully simple and intuitive.
 There are currently no exceptions.
 In particular, there is no special infix operator syntax for arithmetic
 expressions, and therefore no operator precedence rules are necessary.
 You have to write nested arithmetic expressions always in the above prefix
 syntax, like 
\family typewriter
%*{7}{%+{2}{3}}
\family default
 (similar to non-inverse polish notation).
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

When deeply nesting macros and their braces, you may easily find yourself
 in a feeling like in the good old days of Lisp.
 Use the above backslash-newline syntax to indent your macros in a readable
 and structured way.
 Fortunately, modern text editors like (x)emacs or vim have modes for dealing
 with the correctness of nested braces.
\end_layout

\begin_layout Subsection
Calling Builtin / Primitive Macros
\end_layout

\begin_layout Standard
Primitive macros can be called in two alternate forms:
\end_layout

\begin_layout Itemize

\family typewriter
%primitive-
\emph on
macroname
\emph default
{
\emph on
something
\emph default
}
\end_layout

\begin_layout Itemize

\family typewriter
%
\emph on
macroname
\emph default
{
\emph on
something
\emph default
}
\end_layout

\begin_layout Standard
When using the 
\family typewriter
%primitive-*{}
\family default
 form, you 
\emph on
explicitly disallow
\emph default
 interception of the call by a 
\family typewriter
*.tpl
\family default
 file.
 Otherwise, you may override the standard definition even of primitive macros
 by your own template files.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Notice that 
\family typewriter
%call{}
\family default
 conventions are used in such a case.
 The parameters are passed via 
\family typewriter
%{0}
\family default
 
\begin_inset Formula $\ldots$
\end_inset


\family typewriter
%{n}
\family default
 variables (see description below).
\end_layout

\begin_layout Paragraph
Standard MARS State Inspection Macros
\end_layout

\begin_layout Standard
These are already described in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Predefined-Trivial-Macros"

\end_inset

.
 When calling one of them, the call will simply expand to the corresponding
 value.
\end_layout

\begin_layout Standard
Example: 
\family typewriter
%get-primary{}
\family default
 will expand to the hostname of the current designated primary node.
\end_layout

\begin_layout Paragraph
Further MARS State Inspection Macros
\end_layout

\begin_layout Paragraph
Variable Access Macros
\end_layout

\begin_layout Itemize

\family typewriter
%let{
\emph on
varname
\emph default
}{
\emph on
expression
\emph default
} 
\family default
Evaluates both 
\family typewriter
\emph on
varname
\family default
\emph default
 and the 
\family typewriter
\emph on
expression
\family default
\emph default
.
 The 
\family typewriter
\emph on
expression
\family default
\emph default
 is then assigned to 
\family typewriter
varname
\family default
.
\end_layout

\begin_layout Itemize

\family typewriter
%let{
\emph on
varname
\emph default
}{
\emph on
expression
\emph default
} 
\family default
Evaluates both 
\family typewriter
\emph on
varname
\family default
\emph default
 and the 
\family typewriter
\emph on
expression
\family default
\emph default
.
 The 
\family typewriter
\emph on
expression
\family default
\emph default
 is then appended to 
\family typewriter
varname
\family default
 (concatenation).
\end_layout

\begin_layout Itemize

\family typewriter
%{
\emph on
varname
\emph default
} 
\family default
Evaluates 
\family typewriter
\emph on
varname
\family default
\emph default
, and outputs the value of the corresponding variable.
 When the variable does not exist, the empty string is returned.
\end_layout

\begin_layout Itemize

\family typewriter
%{++}{
\emph on
varname
\emph default
} 
\family default
or 
\family typewriter
%{
\emph on
varname
\emph default
}{++}
\family default
 Has the obvious well-known side effect e.g.
 from C or Java.
 You may also use 
\family typewriter
--
\family default
 instead of 
\family typewriter
++
\family default
.
 This is handy for programming loops (see below).
\end_layout

\begin_layout Itemize

\family typewriter
%dump-vars{} 
\family default
Writes all currently defined variables (from the currently active scope)
 to 
\family typewriter
stderr
\family default
.
 This is handy for debugging.
\end_layout

\begin_layout Paragraph
CSV Array Macros
\end_layout

\begin_layout Itemize

\family typewriter
%{
\emph on
varname
\emph default
}{
\emph on
delimiter
\emph default
}{
\emph on
index
\emph default
} 
\family default
Evaluates all arguments.
 The contents of 
\family typewriter
\emph on
varname
\family default
\emph default
 is interpreted as a comma-separated list, delimited by 
\family typewriter
\emph on
delimiter
\family default
\emph default
.
 The 
\family typewriter
\emph on
index
\family default
\emph default
'th list element is returned.
\end_layout

\begin_layout Itemize

\family typewriter
%set{
\emph on
varname
\emph default
}{
\emph on
delimiter
\emph default
}{
\emph on
index
\emph default
}{
\emph on
expression
\emph default
} 
\family default
Evaluates all arguments.
 The contents of the old 
\family typewriter
\emph on
varname
\family default
\emph default
 is interpreted as a comma-separated list, delimited by 
\family typewriter
\emph on
delimiter
\family default
\emph default
.
 The 
\family typewriter
\emph on
index
\family default
\emph default
'th list element is the assigend to, or substituted by, 
\family typewriter
\emph on
expression
\family default
\emph default
.
\end_layout

\begin_layout Paragraph
Arithmetic Expression Macros
\end_layout

\begin_layout Standard
The following macros can also take more than two arguments, carrying out
 the corresponding arithmetic operation in sequence (it depends on the operator
 whether this accords to the associative law).
\end_layout

\begin_layout Itemize

\family typewriter
%+{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Evaluates the arguments, inteprets them as numbers, and adds them together.
\end_layout

\begin_layout Itemize

\family typewriter
%-{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Subtraction.
\end_layout

\begin_layout Itemize

\family typewriter
%*{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Multiplication.
\end_layout

\begin_layout Itemize

\family typewriter
%/{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Division.
\end_layout

\begin_layout Itemize

\family typewriter
%%{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Modulus.
\end_layout

\begin_layout Itemize

\family typewriter
%&{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Bitwise Binary And.
\end_layout

\begin_layout Itemize

\family typewriter
%|{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Bitwise Binary Or.
\end_layout

\begin_layout Itemize

\family typewriter
%^{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Bitwise Binary Exclusive Or.
\end_layout

\begin_layout Itemize

\family typewriter
%<<{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Binary Shift Left.
\end_layout

\begin_layout Itemize

\family typewriter
%>>{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Binary Shift Right.
\end_layout

\begin_layout Itemize

\family typewriter
%min{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Compute the arithmetic minimum of the arguments.
\end_layout

\begin_layout Itemize

\family typewriter
%max{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Compute the arithmetic maximum of the arguments.
\end_layout

\begin_layout Paragraph
Boolean Condition Macros
\end_layout

\begin_layout Itemize

\family typewriter
%=={
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Numeral Equality.
\end_layout

\begin_layout Itemize

\family typewriter
%!={
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Numeral Inequality.
\end_layout

\begin_layout Itemize

\family typewriter
%<{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Numeral Less Then.
\end_layout

\begin_layout Itemize

\family typewriter
%<={
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Numeral Less or Equal.
\end_layout

\begin_layout Itemize

\family typewriter
%>{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Numeral Greater Then.
\end_layout

\begin_layout Itemize

\family typewriter
%>={
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Numeral Greater or Equal.
\end_layout

\begin_layout Itemize

\family typewriter
%eq{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default

\begin_inset space ~
\end_inset

String Equality.
\end_layout

\begin_layout Itemize

\family typewriter
%ne{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
String Inequality.
\end_layout

\begin_layout Itemize

\family typewriter
%lt{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
String Less Then.
\end_layout

\begin_layout Itemize

\family typewriter
%le{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
String Less or Equal.
\end_layout

\begin_layout Itemize

\family typewriter
%gt{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
String Greater Then.
\end_layout

\begin_layout Itemize

\family typewriter
%ge{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
String Greater or Equal.
\end_layout

\begin_layout Itemize

\family typewriter
%=~{
\emph on
string
\emph default
}{
\emph on
regex
\emph default
}{
\emph on
opts
\emph default
} 
\family default
or 
\family typewriter
%match{
\emph on
string
\emph default
}{
\emph on
regex
\emph default
}{
\emph on
opts
\emph default
}
\family default
 Checks whether 
\family typewriter
\emph on
string
\family default
\emph default
 matches the Perl regular expression 
\family typewriter
\emph on
regex
\family default
\emph default
.
 Modifiers can be given via 
\family typewriter
\emph on
opts
\family default
\emph default
.
\end_layout

\begin_layout Paragraph
Shortcut Evaluation Operators
\end_layout

\begin_layout Standard
The following operators evaluate their arguments only when needed (like
 in C).
\end_layout

\begin_layout Itemize

\family typewriter
%&&{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Logical And.
\end_layout

\begin_layout Itemize

\family typewriter
%and{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Alias for 
\family typewriter
%&&{}
\family default
.
\end_layout

\begin_layout Itemize

\family typewriter
%||{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Logical Or.
\end_layout

\begin_layout Itemize

\family typewriter
%or{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
} 
\family default
Alias for 
\family typewriter
%||{}
\family default
.
\end_layout

\begin_layout Paragraph
Unary Operators
\end_layout

\begin_layout Itemize

\family typewriter
%!{
\emph on
arg
\emph default
} 
\family default
Logical Not.
\end_layout

\begin_layout Itemize

\family typewriter
%not{
\emph on
arg
\emph default
} 
\family default
Alias for 
\family typewriter
%!{}
\family default
.
\end_layout

\begin_layout Itemize

\family typewriter
%~{
\emph on
arg
\emph default
} 
\family default
Bitwise Ńegation.
\end_layout

\begin_layout Paragraph
String Functions
\end_layout

\begin_layout Itemize

\family typewriter
%length{
\emph on
string
\emph default
} 
\family default
Return the number of ASCII characters present in 
\family typewriter
\emph on
string
\family default
\emph default
.
\end_layout

\begin_layout Itemize

\family typewriter
%toupper{
\emph on
string
\emph default
} 
\family default
Return all ASCII characters converted to uppercase.
\end_layout

\begin_layout Itemize

\family typewriter
%tolower{
\emph on
string
\emph default
} 
\family default
Return all ASCII characters converted to lowercase.
\end_layout

\begin_layout Itemize

\family typewriter
%append{
\emph on
varname
\emph default
}{
\emph on
string
\emph default
} 
\family default
Equivalent to 
\family typewriter
%let{
\emph on
varname
\emph default
}{%{
\emph on
varname
\emph default
}
\emph on
string
\emph default
}
\family default
.
\end_layout

\begin_layout Itemize

\family typewriter
%subst{
\emph on
string
\emph default
}{
\emph on
regex
\emph default
}{
\emph on
subst
\emph default
}{
\emph on
opts
\emph default
} 
\family default
Perl regex substitution.
\end_layout

\begin_layout Itemize

\family typewriter
%sprintf{
\emph on
fmt
\emph default
}{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
}{
\emph on
argn
\emph default
} 
\family default
Perl 
\family typewriter
sprintf()
\family default
 operator.
 Details see Perl manual.
\end_layout

\begin_layout Itemize

\family typewriter
%human-number{
\emph on
unit
\emph default
}{
\emph on
delim
\emph default
}{
\emph on
unit-sep
\emph default
}{
\emph on
number
\emph default
1}{
\emph on
number
\emph default
2}
\begin_inset Formula $\ldots$
\end_inset

 
\family default
Convert a number or a list of numbers into human-readable 
\family typewriter
B
\family default
, 
\family typewriter
KiB
\family default
, 
\family typewriter
MiB
\family default
, 
\family typewriter
GiB
\family default
, 
\family typewriter
TiB
\family default
, as given by 
\family typewriter
\emph on
unit
\family default
\emph default
.
 When 
\family typewriter
\emph on
unit
\family default
\emph default
 is empty, a reasonable unit will be guessed automatically from the maximum
 of all given numbers.
 A single result string is produced, where multiple numbers are separated
 by 
\family typewriter
\emph on
delim
\family default
\emph default
 when necessary.
 When 
\family typewriter
\emph on
delim
\family default
\emph default
 is empty, the slash symbol 
\family typewriter
/
\family default
 is used by default (the most obvious use case is result strings like 
\family typewriter

\begin_inset Quotes eld
\end_inset

17/32 KiB
\begin_inset Quotes erd
\end_inset


\family default
).
 The final unit text is separated from the previous number(s) by 
\family typewriter
\emph on
unit-sep
\family default
\emph default
.
 When 
\family typewriter
\emph on
unit-sep
\family default
\emph default
 is empty, a single blank is used by default.
\end_layout

\begin_layout Itemize

\family typewriter
%human-seconds{
\emph on
number
\emph default
} 
\family default
Convert the given number of seconds into 
\family typewriter
hh:mm:ss
\family default
 format.
\end_layout

\begin_layout Paragraph
Complex Helper Macros
\end_layout

\begin_layout Itemize

\family typewriter
%progress{20} 
\family default
Return a string containing a progress bar showing the values from 
\family typewriter
%summary-vector{}
\family default
.
 The default width is 20 characters plus two braces.
\end_layout

\begin_layout Itemize

\family typewriter
%progress{20}{
\emph on
minvalue
\emph default
}{
\emph on
midvalue
\emph default
}{
\emph on
maxvalue
\emph default
} 
\family default
Instead of taking the values from 
\family typewriter
%summary-vector{}
\family default
, use the supplied values.
 
\family typewriter
minvalue
\family default
 and 
\family typewriter
midvalue
\family default
 indicate two different intermediate points, while 
\family typewriter
maxvalue
\family default
 will determine the 100% point.
\end_layout

\begin_layout Paragraph
Control Flow Macros
\end_layout

\begin_layout Itemize

\family typewriter
%if{
\emph on
expression
\emph default
}{
\emph on
then-part
\emph default
}
\family default
 or 
\family typewriter
%if{
\emph on
expression
\emph default
}{
\emph on
then-part
\emph default
}{
\emph on
else-part
\emph default
}
\family default
 Like in any other macro or programming language, this evaluates the 
\family typewriter
expression
\family default
 once, not copying its outcome to the output.
 If the result is non-empty and is not a string denoting the number 
\family typewriter
0
\family default
, the 
\family typewriter
\emph on
then-part
\family default
\emph default
 is evaluated and copied to the output.
 Otherwise, the 
\family typewriter
else-part
\family default
 is evaluated and copied, provided that one exists.
\end_layout

\begin_layout Itemize

\family typewriter
%unless{
\emph on
expression
\emph default
}{
\emph on
then-part
\emph default
}
\family default
 or 
\family typewriter
%unless{
\emph on
expression
\emph default
}{
\emph on
then-part
\emph default
}{
\emph on
else-part
\emph default
}
\family default
 Like 
\family typewriter
%if{}
\family default
, but the expression is logically negated.
 Essentially, this is a shorthand for 
\family typewriter
%if{%not{expression}}{...}
\family default
 or similar.
\end_layout

\begin_layout Itemize

\family typewriter
%elsif{
\emph on
expr1
\emph default
}{
\emph on
then1
\emph default
}{
\emph on
expr2
\emph default
}{
\emph on
then2
\emph default
}
\family default

\begin_inset Formula $\ldots$
\end_inset

 or 
\family typewriter
%elsif{
\emph on
expr1
\emph default
}{
\emph on
then1
\emph default
}{
\emph on
expr2
\emph default
}{
\emph on
then2
\emph default
}
\family default

\begin_inset Formula $\ldots$
\end_inset


\family typewriter
{
\emph on
odd-else-part
\emph default
}
\family default
 This is for simplification of boring if-else-if chains.
 The classical if-syntax (as shown above) has the drawback that inner if-parts
 need to be nested into outer else-parts, so rather deep nestings may occur
 when you are programming longer chains.
 This is an alternate syntax for avoidance of deep nesting.
 When giving an odd number of arguments, the last argument is taken as final
 else-part.
\end_layout

\begin_layout Itemize

\family typewriter
%elsunless
\family default

\begin_inset Formula $\ldots$
\end_inset

 Like 
\family typewriter
%elsif
\family default
, but 
\emph on
all
\emph default
 conditions are negated.
\end_layout

\begin_layout Itemize

\family typewriter
%while{
\emph on
expression
\emph default
}{
\emph on
body
\emph default
} 
\family default
Evaluates the 
\family typewriter
\emph on
expression
\family default
\emph default
 in a while loop, like in any other macro or programming language.
 The 
\family typewriter
\emph on
body
\family default
\emph default
 is evaluated exactly as many times as the 
\family typewriter
\emph on
expression
\family default
\emph default
 holds.
 Notice that endless loops can be only avoided by a calling a non-pure macro
 inspecting external state information, or by creating (and checking) another
 side effect somewhere, like assigning to a variable somewhere.
\end_layout

\begin_layout Itemize

\family typewriter
%until{
\emph on
expression
\emph default
}{
\emph on
body
\emph default
} 
\family default
Like
\family typewriter
 %while{
\emph on
expression
\emph default
}{
\emph on
body
\emph default
}
\family default
, but negate the expression.
\end_layout

\begin_layout Itemize

\family typewriter
%for{
\emph on
exp
\emph default
r1}{
\emph on
exp
\emph default
r2}{
\emph on
exp
\emph default
r3}{
\emph on
body
\emph default
}
\family default
 As you will expect from the corresponding C, Perl, Java, or (add your favorite
 language) construct.
 Only the syntactic sugar is a little bit different.
\end_layout

\begin_layout Itemize

\family typewriter
%foreach{
\emph on
varname
\emph default
}{
\emph on
CSV-delimited-string
\emph default
}{
\emph on
delimiter
\emph default
}{
\emph on
body
\emph default
}
\family default
 As you can expect from similar 
\family typewriter
foreach
\family default
 constructs in other languages like Perl.
 Currently, the macro processor has no arrays, but can use comma-separated
 strings as a substitute.
\end_layout

\begin_layout Itemize

\family typewriter
%eval{
\emph on
count
\emph default
}{
\emph on
body
\emph default
}
\family default
 Evaluates the 
\family typewriter
\emph on
body
\family default
\emph default
 exactly as many times as indicated by the numeric argument 
\family typewriter
\emph on
count
\family default
\emph default
.
 This may be used to re-evaluate the output of other macros once again.
\end_layout

\begin_layout Itemize

\family typewriter
%protect{
\emph on
body
\emph default
}
\family default
 Equivalent to 
\family typewriter
%eval{0}{
\emph on
body
\emph default
}
\family default
, which means that the body is not evaluated at all, but copied to the output
 verbatim
\begin_inset Foot
status open

\begin_layout Plain Layout
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
TeX
\end_layout

\end_inset


\begin_inset space ~
\end_inset

or 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
LaTeX
\end_layout

\end_inset


\begin_inset space ~
\end_inset

fans usually know what this is good for ;)
\end_layout

\end_inset

.
\end_layout

\begin_layout Itemize

\family typewriter
%eval-down{
\emph on
body
\emph default
}
\family default
 Evaluates the 
\family typewriter
\emph on
body
\family default
\emph default
 in a loop until the result does not change any more
\begin_inset Foot
status open

\begin_layout Plain Layout
Mathematicians knowing Banach's fixedpoint theorem will know what this is
 good for ;)
\end_layout

\end_inset

.
\end_layout

\begin_layout Itemize

\family typewriter
%tmp{
\emph on
body
\emph default
}
\family default
 Evaluates the 
\family typewriter
\emph on
body
\family default
\emph default
 once in a temporary scope which is thrown away afterwards.
\end_layout

\begin_layout Itemize

\family typewriter
%call{
\emph on
macroname
\emph default
}{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
}{
\emph on
argn
\emph default
}
\family default
 Like in many other macro languages, this evaluates the named macro in the
 a new scope.
 This means that any side effects produced by the called macro, such as
 variable assignments, will be reverted after the call, and therefore not
 influence the old scope.
 However notice that the arguments 
\family typewriter
\emph on
arg1
\family default
\emph default
 to 
\family typewriter
\emph on
argn
\family default
\emph default
 are evaluted in the 
\emph on
old
\emph default
 scope before the call actually happens (possibly producing side effects
 if they contain some), and their result is respectively assigned to 
\family typewriter
%{1}
\family default
 until 
\family typewriter
%{
\emph on
n
\emph default
}
\family default
 in the new scope, analogously to the Shell or to Perl.
 In addition, the new 
\family typewriter
%{0}
\family default
 gets the 
\family typewriter
\emph on
macroname
\family default
\emph default
.
 Notice that the argument evaluation happens non-lazily in the old scope
 and therefore differs from other macro processors like 
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
TeX
\end_layout

\end_inset

.
\end_layout

\begin_layout Itemize

\family typewriter
%include{
\emph on
macroname
\emph default
}{
\emph on
arg1
\emph default
}{
\emph on
arg2
\emph default
}{
\emph on
argn
\emph default
}
\family default
 Like 
\family typewriter
%call{}
\family default
, but evaluates the named macro in the 
\emph on
current
\emph default
 scope (similar to the 
\family typewriter
source
\family default
 command of the bourne shell).
 This means that any side effects produced by the called macro, such as
 variable assignments, will 
\emph on
not
\emph default
 be reverted after the call.
 Even the 
\family typewriter
%{0}
\family default
 until 
\family typewriter
%{
\emph on
n
\emph default
}
\family default
 variables will continue to exist (and may lead to confusion if you aren't
 aware of that).
\end_layout

\begin_layout Itemize

\family typewriter
%callstack{}
\family default
 Useful for debugging: show the current chain of macro invocations.
\end_layout

\begin_layout Paragraph
Time Handling Macros
\end_layout

\begin_layout Itemize

\family typewriter
%time{}
\family default
 Return the current Lamport timestamp (see 
\family typewriter
mars-for-kernel-developers.pdf
\family default
), in units of seconds since the Unix epoch.
\end_layout

\begin_layout Itemize

\family typewriter
%real-time{}
\family default
 Return the current system clock timestamp, in units of seconds since the
 Unix epoch.
\end_layout

\begin_layout Itemize

\family typewriter
%sleep{
\emph on
seconds
\emph default
}
\family default
 Pause the given number of seconds.
\end_layout

\begin_layout Itemize

\family typewriter
%timeout{
\emph on
seconds
\emph default
}
\family default
 Like 
\family typewriter
%sleep{
\emph on
seconds
\emph default
}
\family default
, but abort the 
\family typewriter
marsadm
\family default
 command after the total waiting time has exceeded the timeout given by
 the 
\family typewriter
--timeout=
\family default
 parameter.
\end_layout

\begin_layout Paragraph
Misc Macros
\end_layout

\begin_layout Itemize

\family typewriter
%warn{
\emph on
text
\emph default
}
\family default
 Show a WARNING:
\end_layout

\begin_layout Itemize

\family typewriter
%die{
\emph on
text
\emph default
}
\family default
 Abort execution with an error message.
\end_layout

\begin_layout Paragraph
Experts Only - Risky
\end_layout

\begin_layout Standard
The following macros are unstable and may change at any time without notice.
\end_layout

\begin_layout Itemize

\family typewriter
%get-msg{
\emph on
name
\emph default
}
\family default
 Low-level access to system messages.
 You should not use this, since this is not extensible (you must know the
 name in advance).
\end_layout

\begin_layout Itemize

\family typewriter
%readlink{
\emph on
path
\emph default
}
\family default
 Low-level access to symlinks.
 Don't misuse this for circumvention of the abstraction macros from the
 symlink tree!
\end_layout

\begin_layout Itemize

\family typewriter
%setlink{
\emph on
value
\emph default
}{
\emph on
path
\emph default
}
\family default
 Low-level creation of symlinks.
 Don't misuse this for circumvention of the abstraction macros for the symlink
 tree!
\end_layout

\begin_layout Itemize

\family typewriter
%fetch-info{} 
\family default
etc.
 Low-level access to internal symlink formats.
 Don't use this in scripts! Only for curious humans.
\end_layout

\begin_layout Itemize

\family typewriter
%is-almost-consistent{}
\family default
 Whatever you guess what this could mean, don't use it, at least never in
 place of 
\family typewriter
%is-consistent{}
\family default
 - it is risky to base decisions on this.
 Mostly for historical reasons.
\end_layout

\begin_layout Itemize

\family typewriter
%does{
\emph on
name
\emph default
} 
\family default
Equivalent to 
\family typewriter
%is-
\emph on
name
\emph default
{}
\family default
 (just more handy for computing the macro name).
 Use with care!
\end_layout

\begin_layout Subsection
Predefined Variables
\begin_inset CommandInset label
LatexCommand label
name "par:Predefined-Variables"

\end_inset


\end_layout

\begin_layout Itemize

\family typewriter
%{cmd} 
\family default
The command argument of the invoked 
\family typewriter
marsadm
\family default
 command.
\end_layout

\begin_layout Itemize

\family typewriter
%{res} 
\family default
The resource name given to the 
\family typewriter
marsadm
\family default
 command as a command line parameter (or, possibly expanded from 
\family typewriter
all
\family default
).
\end_layout

\begin_layout Itemize

\family typewriter
%{resdir} 
\family default
The corresponding resource directory.
 The current version of MARS uses 
\family typewriter
/mars/resource-%{res}/
\family default
, but this may change in future.
 Normally, you should not need this, since anything should be already abstracted
 for you.
 In case you 
\emph on
really
\emph default
 need low-level access to something, please prefer this variable over 
\family typewriter
%{mars}/resource-%{res}
\family default
 because it is a bit more abstracted.
\end_layout

\begin_layout Itemize

\family typewriter
%{mars} 
\family default
Currently the fixed string 
\family typewriter
/mars
\family default
.
 This may change in future, probably with the advent of MARS Full.
\end_layout

\begin_layout Itemize

\family typewriter
%{host} 
\family default
The hostname of the local node.
\end_layout

\begin_layout Itemize

\family typewriter
%{ip} 
\family default
The IP address of the local node.
\end_layout

\begin_layout Itemize

\family typewriter
%{timeout} 
\family default
The value given by the 
\family typewriter
--timeout=
\family default
 option, or the corresonding default value.
\end_layout

\begin_layout Itemize

\family typewriter
%{threshold} 
\family default
The value given by the 
\family typewriter
--threshold=
\family default
 option, or the corresonding default value.
\end_layout

\begin_layout Itemize

\family typewriter
%{window} 
\family default
The value given by the 
\family typewriter
--window=
\family default
 option, or the corresonding default value (60s).
\end_layout

\begin_layout Itemize

\family typewriter
%{force} 
\family default
The number of times the 
\family typewriter
--force
\family default
 option has been given.
\end_layout

\begin_layout Itemize

\family typewriter
%{dry-run} 
\family default
The number of times the 
\family typewriter
--dry-run
\family default
 option has been given.
\end_layout

\begin_layout Itemize

\family typewriter
%{verbose} 
\family default
The number of times the 
\family typewriter
--verbose
\family default
 option has been given.
\end_layout

\begin_layout Itemize

\family typewriter
%{callstack} 
\family default
Same as the 
\family typewriter
%callstack{}
\family default
 macro.
 The latter gives you an opportunity for overriding, while the former is
 firmly built in.
\end_layout

\begin_layout Section
Scripting Advice
\begin_inset CommandInset label
LatexCommand label
name "sec:Scripting-HOWTO"

\end_inset


\end_layout

\begin_layout Standard
Both the 
\series bold
asynchronous communication model
\series default
 of MARS including the Lamport clock, and the 
\series bold
state model
\series default
 (cf section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:The-State-of"

\end_inset

) is something you 
\emph on
definitely
\emph default
 should have in mind when you want to do some scripting.
 Here is some advice:
\end_layout

\begin_layout Itemize
Don't access anything on 
\family typewriter
/mars/
\family default
 directly, except for debugging purposes.
 Use 
\family typewriter
marsadm
\family default
.
\end_layout

\begin_layout Itemize
Avoid running scripts in parallel, other than for inspection / monitoring
 purposes.
 When you give two 
\family typewriter
marsadm
\family default
 commands in parallel (whether on the same host, or on different hosts belonging
 to the same cluster), it is possible to produce a mess.
 
\family typewriter
marsadm
\family default
 has no internal locking.
 There is no cluster-wide locking at all, because if would cause trouble
 during long-distance network outages.
 Unfortunately, some systems like Pacemaker are violating this in many cases
 (depending on their configuration).
 Best is if you have a dedicated / more or less centralized 
\series bold
control machine
\series default
 which controls masses of your georedundant working servers.
 This reduces the risk of running interfering actions in parallel.
 Of course, you need backup machines for your control machines, and in different
 locations.
 Not obeying this advice can easily lead to problems such as complex races
 which are very difficult to solve in long-distance distributed systems,
 even in general (not limited to MARS).
\end_layout

\begin_layout Itemize

\family typewriter
marsadm wait-cluster
\family default
 is your friend.
 Whenever your (near-)central script has to switch between different hosts
 
\family typewriter
A
\family default
 and 
\family typewriter
B
\family default
 (of the same cluster), use it in the following way:
\begin_inset Newline newline
\end_inset


\family typewriter
ssh A 
\begin_inset Quotes eld
\end_inset

marsadm action1
\begin_inset Quotes erd
\end_inset

; ssh B 
\begin_inset Quotes eld
\end_inset

marsadm wait-cluster; marsadm action2
\begin_inset Quotes erd
\end_inset


\begin_inset Newline newline
\end_inset


\family default

\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Don't ignore this advice! Interference is almost 
\emph on
sure
\emph default
! As a rule of thumb, precede almost any action command with some appropriate
 waiting command!
\end_layout

\begin_layout Itemize
Further friends are any 
\family typewriter
marsadm wait-*
\family default
 commands, such as 
\family typewriter
wait-umount
\family default
.
\end_layout

\begin_layout Itemize
In some places, busy-wait loops might be needed, e.g.
 for waiting until a specific resource is 
\family typewriter
UpToDate
\family default
 or matches some other condition.
 Examples of waiting conditions can be found under 
\family typewriter
github.com/schoebel/test-suite
\family default
 in subdirectory 
\family typewriter
mars/modules/
\family default
, specifically 
\family typewriter
02_predicates.sh
\family default
 or similar.
\end_layout

\begin_layout Itemize
In case of network problems, some command may hang (forever), if you don't
 set the 
\family typewriter
--timeout=
\family default
 option.
 Don't forget the check the return state of any failed / timeouted commands,
 and to take appropriate measures!
\end_layout

\begin_layout Itemize
Test your scripts in failure scenarios!
\end_layout

\begin_layout Chapter
\start_of_appendix
Technical Data MARS
\begin_inset CommandInset label
LatexCommand label
name "chap:Technical-Data-MARS"

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 Do not use MARS inside of VMs.
 Only use at bare metal!
\end_layout

\begin_layout Standard
MARS  has some built-in limitations which should be overcome
\begin_inset Foot
status open

\begin_layout Plain Layout
Some internal algorithms are quadratic.
 The reason is that MARS  evolved from a lab prototype which wasn't originally
 intended for enterprise grade usage, but should have been succeeded by
 the fully instance-oriented MARS Full much earlier.
\end_layout

\end_inset

 by the future MARS Full.
 Please don't exceed the following limits:
\end_layout

\begin_layout Itemize
maximum 4 nodes per cluster
\end_layout

\begin_layout Itemize
maximum 20 resources per cluster
\end_layout

\begin_layout Itemize
maximum 100 logfiles per resource
\end_layout

\begin_layout Chapter
Handout for Midnight Problem Solving
\begin_inset CommandInset label
LatexCommand label
name "chap:Handout-for-Midnight"

\end_inset


\end_layout

\begin_layout Standard
Here are generic instructions for the generic 
\family typewriter
marsadm
\family default
 and commandline level.
 Other levels (e.g.
 different types of cluster managers, PaceMaker, control scripts / 
\family typewriter
rc
\family default
 scripts / 
\family typewriter
upstart
\family default
 scripts, etc should be described elsewhere.
\end_layout

\begin_layout Section
Inspecting the State of MARS
\end_layout

\begin_layout Standard
For manual inspection, please prefer the new 
\family typewriter
marsadm view all
\family default
 over the old 
\family typewriter
marsadm view-1and1 all
\family default
.
 It shows more appropriate / detailed information.
\end_layout

\begin_layout Standard
Hint: this might change in future when somebody will program better marcros
 for the 
\family typewriter
view-1and1
\family default
 variant, or create even better other macros.
\end_layout

\begin_layout Quotation

\family typewriter
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

# watch marsadm view all
\end_layout

\end_inset


\end_layout

\begin_layout Standard
Checking the low-level network connections at runtime:
\end_layout

\begin_layout Quotation

\family typewriter
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

# watch "netstat --tcp | grep 777"
\end_layout

\end_inset


\end_layout

\begin_layout Standard
Meaning of the port numbers (as currently configured into the kernel module,
 may change in future):
\end_layout

\begin_layout Itemize
7777 = metadata / symlink propagation
\end_layout

\begin_layout Itemize
7778 = transfer of transaction logfiles
\end_layout

\begin_layout Itemize
7779 = transfer of sync traffic
\end_layout

\begin_layout Standard
7777 must be always active on a healthy cluster.
 7778 and 7779 will appear only on demand, when some data is transferred.
\end_layout

\begin_layout Standard
Hint: when one of the columns Send-Q or Recv-Q are constantly at high values,
 you might have a network bottleneck.
\end_layout

\begin_layout Section
Replication is Stuck
\end_layout

\begin_layout Standard
Indications for a stuck: 
\end_layout

\begin_layout Itemize
One of the flags shown by 
\family typewriter
marsadm view all
\family default
 or 
\family typewriter
marsadm view-flags all
\family default
 contain a symbol 
\family typewriter
"-"
\family default
 (dash).
 This means that some switch is currently switched off (deliberately).
 Please check whether there is a valid reason why somebody else switched
 it off.
 If the switch-off is just by accident, use the following command to fix
 the stuck:
\family typewriter

\begin_inset listings
inline false
status open

\begin_layout Plain Layout

# marsadm up all
\end_layout

\end_inset


\family default
(or replace 
\family typewriter
all
\family default
 by a particular resource name if you want to start only a specific one).
\begin_inset Newline newline
\end_inset

Note: 
\family typewriter
up
\family default
 is equivalent to the sequence 
\family typewriter
attach; resume-fetch; resume-replay; resume-sync
\family default
.
 Instead of switching each individual knob, use 
\family typewriter
up
\family default
 as a shortcut for switching on anything which is currently off.
\end_layout

\begin_layout Itemize

\family typewriter
netstat --tcp | grep 7777
\family default
 does not show anything.
 Please check the following:
\begin_inset Separator latexpar
\end_inset


\end_layout

\begin_deeper
\begin_layout Itemize
Is the kernel module loaded? Check 
\family typewriter
lsmod | grep mars
\family default
.
 When necessary, run 
\family typewriter
modprobe mars
\family default
.
\end_layout

\begin_layout Itemize
Is the network interface down? Check 
\family typewriter
ifconfig
\family default
, and/or 
\family typewriter
ethtool
\family default
 and friends, and fix it when necessary.
\end_layout

\begin_layout Itemize
Is a 
\family typewriter
ping <partner-host>
\family default
 possible? If not, fix the network / routing / firewall / etc.
 When fixed, the MARS connections should automatically appear after about
 1 minute.
\end_layout

\begin_layout Itemize
When 
\family typewriter
ping
\family default
 is possible, but a MARS connection to port 7777 does not appear after a
 few minutes, try to connect to remote port 7777 by hand via 
\family typewriter
telnet
\family default
.
 But don't type anything, just abort the connection immediately when it
 works! Typing anything will almost certainly throw a harsh error message
 at the other server, which could unnecessarily alarm other people.
\end_layout

\end_deeper
\begin_layout Itemize
Check whether 
\family typewriter
marsadm view all
\family default
 shows some progress bars somewhere.
 Example:
\family typewriter
\size scriptsize

\begin_inset listings
inline false
status open

\begin_layout Plain Layout

istore-test-bap1:~# marsadm view all
\end_layout

\begin_layout Plain Layout

--------- resource lv-0
\end_layout

\begin_layout Plain Layout

 lv-0 OutDated[F] PausedReplay dCAS-R Secondary istore-test-bs1
\end_layout

\begin_layout Plain Layout

    replaying: [>...................]  1.21%  (12/1020)MiB  logs: [2..3]
\end_layout

\begin_layout Plain Layout

       > fetch: 1008.198 MiB rate: 0 B/sec  remaining: --:--:-- hrs
\end_layout

\begin_layout Plain Layout

       > replay: 0 B rate: 0 B/sec  remaining: 00:00:00 hrs 
\end_layout

\end_inset


\family default
\size default
At least one of the 
\family typewriter
rate:
\family default
 values should be greater than 0.
 When none of the 
\family typewriter
rate:
\family default
 values indicate any progress for a longer time, try 
\family typewriter
marsadm up all
\family default
 again.
 If it doesn't help, check and repair the network.
 If even this does not help, check the hardware for any IO hangups, or kernel
 hangups.
 First, check the RAID controllers.
 Often (but not certainly), a stuck kernel can be recognized when many processes
 are 
\emph on
permanently
\emph default
 in state "D", for a long time: 
\family typewriter
ps ax | grep " D" | grep -v grep
\family default
 or similar.
 Please check whether there is just an overload, or 
\emph on
really
\emph default
 a true kernel problem.
 Discrimination is not easy, and requires experience (as with any other
 system; not limited to MARS).
 A truly stuck kernel can only be resurrected by rebooting.
 The same holds for any hardware problems.
\end_layout

\begin_layout Itemize
Check whether 
\family typewriter
marsadm view all
\family default
 reports any lines like 
\family typewriter
WARNING: SPLIT BRAIN at '' detected
\family default
.
 In such a case, check that there is 
\emph on
really
\emph default
 a split brain, before obeying the instructions in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Resolution-of-Split"

\end_inset

.
 Notice that network outages or missing 
\family typewriter
marsadm log-delete-all all
\family default
 or 
\family typewriter
cron
\family default
 may continue to report an old split brain which has gone in the meantime.
\end_layout

\begin_layout Itemize
Check whether 
\family typewriter
/mars/
\family default
 is too full.
 For a rough impression, 
\family typewriter
df /mars/
\family default
 may be used.
 For getting authoritative values as internally used by the MARS emergency-mode
 computations, use 
\family typewriter
marsadm view-rest-space
\family default
 (the unit is GiB).
 In practice, the differences are only marginal, at least on bigger 
\family typewriter
/mars/
\family default
 partitions.
 When there is only few rest space (or none at all), please obey the instruction
s in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Resolution-of-Emergency"

\end_inset

.
\end_layout

\begin_layout Section
Resolution of Emergency Mode
\begin_inset CommandInset label
LatexCommand label
name "sec:Resolution-of-Emergency"

\end_inset


\end_layout

\begin_layout Standard
Emergency mode occurs when 
\family typewriter
/mars/
\family default
 runs out of space, such that no new logfile data can be written anymore.
\end_layout

\begin_layout Standard
In emergency mode, the primary will write any write requests 
\emph on
directly
\emph default
 to the underlying disk, as if MARS were not present at all.
 Thus, your application will continue to run.
 Only the 
\emph on
replication
\emph default
 as such is stopped.
\end_layout

\begin_layout Standard
\begin_inset Note Greyedout
status open

\begin_layout Plain Layout
Notice: emergency mode means that your secondary nodes are usually in a
 
\emph on
consistent
\emph default
, but 
\emph on
outdated
\emph default
 state (exception: when a sync was running in parallel to the emergency
 mode, then the sync will be automatically started over again).
 You can check consistency via 
\family typewriter
marsadm view-flags all
\family default
.
 Only when a local disk shows a lower-case letter 
\family typewriter
"d"
\family default
 instead of an uppercase 
\family typewriter
"D"
\family default
, it is known to be inconsistent (e.g.
 during a sync).
 When there is a dash instead, it usually means that the disk is detatched
 or misconfigured or the kernel module is not started.
 Please fix these problems first before believing that your local disk is
 unusable.
 Even if it is really inconsistent (which is very unlikely, typically occurring
 only as a consequence of hardware failures, or of the above-mentioned exception
), you have a big chance to recover most of the data via 
\family typewriter
fsck
\family default
 and friends.
\end_layout

\end_inset


\end_layout

\begin_layout Standard
A currently existing Emergency mode can be detected by 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

primary:~# marsadm view-is-emergency all
\end_layout

\begin_layout Plain Layout

secondary:~# marsadm view-is-emergency all
\end_layout

\end_inset

 Notice: this delivers the current state, telling nothing about the past.
\end_layout

\begin_layout Standard
Currently, emergency mode will also show something like 
\family typewriter
WARNING: SPLIT BRAIN at '' detected
\family default
.
 This ambiguity will be resolved in a future MARS release.
 It is however not crucial: the resolution methods for both cases are very
 similar.
 If in doubt, start emergency resolution first, and only proceed to split
 brain resoultion if it did not help.
\end_layout

\begin_layout Standard
Preconditions:
\end_layout

\begin_layout Itemize
Only current version of MARS: the space at the primary side should have
 been already released, and the emergency mode should have been already
 left.
 Otherwise, you might need the split-brain resolution method from section
 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Resolution-of-Split"

\end_inset

.
\end_layout

\begin_layout Itemize
The network 
\series bold
must
\series default
 be working.
 Check that the following gives an entry for each secondary: 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

primary:~# netstat --tcp | grep 7777
\end_layout

\end_inset

When necessary, fix the network first (see instructions above).
\end_layout

\begin_layout Standard
Emergency mode should now be resolved via the following instructions: 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

primary:~# marsadm view-is-emergency all
\end_layout

\begin_layout Plain Layout

primary:~# du -s /mars/resource-* | sort -n
\end_layout

\end_inset

Remember the affected resources.
 Best practice is to do the following, starting with the 
\emph on
biggest
\emph default
 resource as shown by the 
\family typewriter
du | sort
\family default
 output in reverse order, but 
\emph on
starting
\emph default
 the following only with the 
\emph on
affected
\emph default
 resources in the first place: 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

secondary1:~# marsadm invalidate <res1>
\end_layout

\begin_layout Plain Layout

secondary1:~# marsadm log-delete-all all
\end_layout

\begin_layout Plain Layout

...
 dito with all resources showing emergency mode
\end_layout

\begin_layout Plain Layout

...
 dito on all other secondaries
\end_layout

\begin_layout Plain Layout

primary:~# marsadm log-delete-all all
\end_layout

\end_inset


\end_layout

\begin_layout Standard
Hint: during the resolution process, some other resources might have gone
 into emergency mode concurrently.
 In addition, it is possible that some secondaries are stuck at particular
 resources while the corresponding primary has 
\emph on
not yet
\emph default
 entered emergency mode.
 Please repeat the steps in such a case, and look for emergency modes at
 secondaries additionally.
 When necessary, extend your list of 
\emph on
affected
\emph default
 resources.
\end_layout

\begin_layout Standard
Hint: be patient.
 Deleting large bulks of logfile data may take a long time, at least on
 highly loaded systems.
 You should give the cleanup processes at least 5 minutes before concluding
 that an 
\family typewriter
invalidate
\family default
 followed by 
\family typewriter
log-delete-all
\family default
 had no effect! Don't forget to give the 
\family typewriter
log-delete-all
\family default
 at all cluster nodes, even when seemingly unaffected.
\end_layout

\begin_layout Standard
In very complex scenarios, when the primary roles of different resources
 are spread over diffent hosts (aka mixed operation), you may need to repeat
 the whole cycle iteratively for a few cycles until the jam is resolved.
\end_layout

\begin_layout Standard
If it does not go away, you have another chance by the following split-brain
 resolution process, which will also cleanup emergency mode as a side effect.
\end_layout

\begin_layout Section
Resolution of Split Brain and of Emergency Mode
\begin_inset CommandInset label
LatexCommand label
name "sec:Resolution-of-Split"

\end_inset


\end_layout

\begin_layout Standard
Hint: in many cases (but not guaranteed), the previous receipe for resolution
 of emergency mode will also cleanup split brain.
 Good chances are in case of 
\begin_inset Formula $k=2$
\end_inset

 total replicas.
 Please collect your own experiences which method works better for you!
\end_layout

\begin_layout Standard
Precondition: the network must be working.
 Check that the following gives an entry for each secondary:
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

primary:~# netstat --tcp | grep 7777
\end_layout

\end_inset

 When necessary, fix the network first (see instructions above).
\end_layout

\begin_layout Standard
Inspect the split brain situation: 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

primary:~# marsadm view all
\end_layout

\begin_layout Plain Layout

primary:~# du -s /mars/resource-* | sort -n
\end_layout

\end_inset

Remember those resources where a message like 
\family typewriter
WARNING: SPLIT BRAIN at '' detected
\family default
 appears.
 Do the following only for 
\emph on
affected
\emph default
 resources, starting with the biggest one (before proceeding to the next
 one).
\end_layout

\begin_layout Standard
Do the following with only 
\emph on
one
\emph default
 resource at a time (before proceeding to the next one), and repeat the
 actions on that resource at every secondary (if there are multiple secondaries)
: 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

secondary1:~# marsadm leave-resource $res1
\end_layout

\begin_layout Plain Layout

secondary1:~# marsadm log-delete-all all
\end_layout

\end_inset

Check whether the split brain has vanished everywhere.
 Startover with other resources at their secondaries when necessary.
\end_layout

\begin_layout Standard
Finally, when no split brain is reported at any (former) secondary, do the
 following on the primary: 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

primary:~# marsadm log-delete-all all
\end_layout

\begin_layout Plain Layout

primary:~# sleep 30
\end_layout

\begin_layout Plain Layout

primary:~# marsadm view all
\end_layout

\end_inset

 Now, the split brain should be gone even at the primary.
 If not, repeat this step.
\end_layout

\begin_layout Standard
In case even this should fail on some 
\family typewriter
$res
\family default
 (which is very unlikely), read the PDF manual before using 
\family typewriter
marsadm log-purge-all $res
\family default
.
 
\end_layout

\begin_layout Standard
Finally, when the split brain is gone everywhere, rebuild the redundancy
 at every secondary via 
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

secondary1:~# marsadm join-resource $res1 /dev/<lv-x>/$res1
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\noindent
If even this method does not help, setup the whole cluster afresh by 
\family typewriter
rmmod mars
\family default
 everywhere, and creating a fresh 
\family typewriter
/mars/
\family default
 filesystem everywhere, followed by the same procedure as installing MARS
 for the first time (which is outside the scope of this handout).
\end_layout

\begin_layout Section
Handover of Primary Role
\end_layout

\begin_layout Standard
When there exists a method for primary handover in higher layers such as
 cluster managers, please prefer that method (e.g.
 
\family typewriter
cm3
\family default
 or other tools).
\end_layout

\begin_layout Standard
If suchalike doesn't work, or if you need to handover some resource 
\family typewriter
$res1
\family default
 by hand, do the following:
\end_layout

\begin_layout Itemize
Stop the load / application corresponding to 
\family typewriter
$res1
\family default
 on the old primary side.
\end_layout

\begin_layout Itemize

\family typewriter
umount /dev/mars/$res1
\family default
, or otherwise close any openers such as iSCSI.
\end_layout

\begin_layout Itemize
At the new primary: 
\family typewriter
marsadm primary $res1
\end_layout

\begin_layout Itemize
Restart the application at the new site (in reverse order to above).
 In case you want to switch 
\emph on
all
\emph default
 resources which are not yet at the new side, you may use 
\family typewriter
marsadm primary all
\family default
.
\end_layout

\begin_layout Section
Emergency Switching of Primary Role
\end_layout

\begin_layout Standard
Emergency switching is necessary when your primary is no longer reachable
 over the network for a 
\emph on
longer
\emph default
 time, or when the hardware is defective.
\end_layout

\begin_layout Standard
Emergency switching will very often lead to a split brain, which requires
 lots of manual actions to resolve (see above).
 Therefore, try to avoid emergency switching when possible!
\end_layout

\begin_layout Standard
Hint: MARS can automatically recover after a primary crash / reboot, as
 well as after secondary crashes, just by executing 
\family typewriter
modprobe mars
\family default
 after 
\family typewriter
/mars/
\family default
 had been mounted.
 Please consider to wait until your system comes up again, instead of risking
 a split brain.
\end_layout

\begin_layout Standard
The decision between emergency switching and continuing operation at the
 same primary side is an operational one.
 MARS can support your decision by the following information at the potentially
 new primary side (which was in secondary mode before): 
\family typewriter
\size scriptsize

\begin_inset listings
inline false
status open

\begin_layout Plain Layout

istore-test-bap1:~# marsadm view all
\end_layout

\begin_layout Plain Layout

--------- resource lv-0
\end_layout

\begin_layout Plain Layout

lv-0 InConsistent Syncing dcAsFr Secondary istore-test-bs1
\end_layout

\begin_layout Plain Layout

syncing: [====>..............] 27.84% (567/2048)MiB rate: 72583.00 KiB/sec remaining: 00:00:20
 hrs
\end_layout

\begin_layout Plain Layout

> sync: 567.293/2048 MiB rate: 72583 KiB/sec remaining: 00:00:20 hrs
\end_layout

\begin_layout Plain Layout

replaying: [>:::::::::::::::::::] 0.00% (0/12902)KiB logs: [1..1]
\end_layout

\begin_layout Plain Layout

> fetch: 0 B rate: 38 KiB/s remaining: 00:00:00
\end_layout

\begin_layout Plain Layout

> replay: 12902.047 KiB rate: 0 B/s remaining: --:--:--
\end_layout

\end_inset


\family default
\size default
When your target is syncing (like in this example), you cannot switch to
 it (same as with DRBD).
 When you had an emergency mode before, you should first resolve that (whenever
 possible).
 When a split brain is reported, try to resolve it first (same as with DRBD).
 Only in case you 
\emph on
know
\emph default
 that the primary is really damaged, or it is really impossible to the run
 the application there for some reason, emergency switching is desirable.
\end_layout

\begin_layout Standard
Hint: in case the secondary is inconsistent for some reason, e.g.
 because of an incremental fast full-sync, you have a last chance to recover
 most data after forceful switching by using a filesystem check or suchalike.
 This might be even faster than restoring data from the backup.
 But use it only if you are 
\emph on
really
\emph default
 desperate!
\end_layout

\begin_layout Standard
The amount of data which is 
\emph on
known
\emph default
 to be missing at your secondary is shown after the 
\family typewriter
> fetch:
\family default
 in human-readable form.
 However, in cases of networking problems this information may be outdated.
 You 
\emph on
always
\emph default
 need to consider further facts which cannot be known by MARS.
\end_layout

\begin_layout Standard
When there exists a method for emergency switching of the primary in higher
 layers such as cluster managers, please prefer that method in front of
 the following one.
\end_layout

\begin_layout Standard
If suchalike doesn't work, or when a handover attempt has failed several
 times, or if you 
\emph on
really need
\emph default
 forceful switching of some resource 
\family typewriter
$res1
\family default
 by hand, you can do the following:
\end_layout

\begin_layout Itemize
When possible, stop the load / application corresponding to 
\family typewriter
$res1
\family default
 on the old primary side.
\end_layout

\begin_layout Itemize
When possible, 
\family typewriter
umount /dev/mars/$res1
\family default
, or otherwise close any openers such as iSCSI.
\end_layout

\begin_layout Itemize
When possible (if you have some time), wait until as much data has been
 propagated to the new primary as possible (watch the 
\family typewriter
fetch:
\family default
 indicator).
\end_layout

\begin_layout Itemize
At the new primary: 
\family typewriter
marsadm disconnect $res1; marsadm primary --force $res1
\end_layout

\begin_layout Itemize
Restart the application at the new site (in reverse order to above).
\end_layout

\begin_layout Itemize
After the application is known to run reliably, check for split brains and
 cleanup them when necessary.
\end_layout

\begin_layout Chapter
Alternative Methods for Split Brain Resolution
\begin_inset CommandInset label
LatexCommand label
name "chap:Alternative-Methods-for"

\end_inset


\end_layout

\begin_layout Standard
Instead of 
\family typewriter
marsadm invalidate
\family default
, the following steps may be used.
 In preference, start with the old 
\begin_inset Quotes eld
\end_inset

wrong
\begin_inset Quotes erd
\end_inset

 primaries first:
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm leave-resource mydata
\end_layout

\begin_layout Enumerate
After having done this on one cluster node, check whether the split brain
 is already gone (e.g.
 by saying 
\family typewriter
marsadm view mydata
\family default
).
 There are chances that you don't need this on all of your nodes.
 Only in very rare
\begin_inset Foot
status open

\begin_layout Plain Layout
When your network had partitioned in a very awkward way for a long time,
 and when your partitioned primaries did several 
\family typewriter
log-rotate
\family default
 operations indendently from each other, there is a small chance that 
\family typewriter
leave-resource
\family default
 does not clean up 
\emph on
all
\emph default
 remains of such an awkward situation.
 Only in such a case, try 
\family typewriter
log-purge-all
\family default
.
\end_layout

\end_inset

 cases, it might happen that the preceding l
\family typewriter
eave-resource
\family default
 operations were not able to clean up all logfiles produced in parallel
 by the split brain situation.
 
\end_layout

\begin_layout Enumerate
Read the documentation about 
\family typewriter
log-purge-all
\family default
 (see page 
\begin_inset CommandInset ref
LatexCommand pageref
reference "log-purge-all$res"

\end_inset

) and use it.
\end_layout

\begin_layout Enumerate
If you want to restore redundancy, you can follow-up a 
\family typewriter
join-resource
\family default
 phase to the old resource name (using the correct device name, double-check
 it!) This will restore your redundancy by overwriting your bad split brain
 version with the correct one.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

It is important to resolve the split brain 
\emph on
before
\emph default
 you can start the 
\family typewriter
join-resource
\family default
 reconstruction phase! In order to keep as many 
\begin_inset Quotes eld
\end_inset

good
\begin_inset Quotes erd
\end_inset

 versions as possible (e.g.
 for emergency cases), don't re-join them all in parallel, but rather start
 with the oldest / most outdated / worst / inconsistent version first.
 It is recommended to start the next one only when the previous one has
 sucessfully finished.
\end_layout

\begin_layout Chapter
Alternative De- and Reconstruction of a Damaged Resource
\begin_inset CommandInset label
LatexCommand label
name "chap:Alternative-De--and"

\end_inset


\end_layout

\begin_layout Standard
In case 
\family typewriter
leave-resource --host=
\family default
 does not work, you may use the following fallback.
 On the surviving new designated primary, give the following commands:
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm disconnect-all mydata
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm down mydata
\end_layout

\begin_layout Enumerate
Check by hand whether your local disk is consistent, e.g.
 by test-mounting it readonly, 
\family typewriter
fsck
\family default
, etc.
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm delete-resource mydata
\end_layout

\begin_layout Enumerate
Check whether the other vital cluster nodes don't report the dead resource
 any more, e.g.
 
\family typewriter
marsadm view all
\family default
 at 
\emph on
each
\emph default
 of them.
 In case the resource has not disappeared anywhere (which may happen during
 network problems), do the 
\family typewriter
down ; delete-resource
\family default
 steps also there (optionally again with 
\family typewriter
--force
\family default
).
\end_layout

\begin_layout Enumerate
Be sure that the resource has disappeared 
\emph on
everywhere
\emph default
.
 When necessary, repeat the 
\family typewriter
delete-resource
\family default
 with 
\family typewriter
--force
\family default
.
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm create-resource newmydata ...

\family default
 at the 
\emph on
correct
\emph default
 node using the 
\emph on
correct
\emph default
 disk device containing the 
\emph on
correct
\emph default
 version, and further steps to setup your resource from scratch, preferably
 under a different name to minimize any risk.
\end_layout

\begin_layout Standard
\noindent
In any case, 
\series bold
manually check
\series default
 whether a split brain is reported for any resource on any of your 
\emph on
surviving
\emph default
 cluster nodes.
 If you find one there (and only then), please (re-)execute the split brain
 resolution steps on the affected node(s).
\end_layout

\begin_layout Chapter
Cleanup in case of Complicated Cascading Failures
\begin_inset CommandInset label
LatexCommand label
name "subsec:Cleanup-in-case"

\end_inset


\end_layout

\begin_layout Standard
MARS does its best to recover even from multiple failures (e.g.
 
\series bold
rolling disasters
\series default
).
 Chances are high that the instructions from sections 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Split-Brain-Resolution"

\end_inset

 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Final-Destroy-of"

\end_inset

 or appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Alternative-Methods-for"

\end_inset

 
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Alternative-De--and"

\end_inset

 will work even in case of multiple failures, such as a network failure
 plus local node failure at only 1 node (even if that node is the former
 primary node).
\end_layout

\begin_layout Standard
However, in general (e.g.
 when more than 1 node is damaged and/or when the filesystem 
\family typewriter
/mars/
\family default
 is badly damaged) there is no general guarantee that recovery will 
\emph on
always
\emph default
 succeed under 
\emph on
any
\emph default
 (weird) circumstances.
 That said, your chances for recovery are 
\emph on
very
\emph default
 high when some disk remains usable at least at one of your surviving secondarie
s.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

It should be very hard to finally trash a secondary, because the transaction
 logfiles are containing 
\family typewriter
md5
\family default
 checksums for all data records.
 Any attempt to replay currupted logfiles is refused by MARS.
 In addition, the sequence numbers of 
\family typewriter
log-rotate
\family default
d logfiles are checked for contiguity.
 Finally, the 
\emph on
sequence path
\emph default
 of logfile applications (consisting of logfile names plus their respective
 length) is additionally secured by a 
\family typewriter
git
\family default
-like incremental checksum over the whole path history (so-called 
\begin_inset Quotes eld
\end_inset

version links
\begin_inset Quotes erd
\end_inset

).
 This should detect split brains even if logfiles are appended / modified
 
\emph on
after
\emph default
 a (forceful) switchover has already taken place.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresToxiques.png
	lyxscale 50
	scale 17

\end_inset

 That said, your risk of final data loss is very high if you remove the
 
\series bold
BBU
\series default
 from your hardware RAID controller before all hot data has been flushed
 to the physical disks.
 Therefore, never try to 
\begin_inset Quotes eld
\end_inset

repair
\begin_inset Quotes erd
\end_inset

 a seemingly dead node before your replication is up again somewhere else!
 Only unplug the network cables when advised, but never try to repair the
 hardware instantly!
\end_layout

\begin_layout Standard
In case of desperate situations where none of the previous instructions
 have succeeded, your last chance is rebuilding all your resources from
 intact disks as follows:
\end_layout

\begin_layout Enumerate
Do 
\family typewriter
rmmod mars
\family default
 on all your cluster nodes and/or reboot them.
 Note: if you are less desperate, chances are high that the following will
 also work when the kernel module remains active and everywhere a 
\family typewriter
marsadm down
\family default
 is given instead, but for an 
\emph on
ultimate
\emph default
 instruction you should eliminate 
\emph on
potential
\emph default
 kernel problems by 
\family typewriter
rmmod
\family default
 / 
\family typewriter
reboot
\family default
, at least if you can afford the downtime on concurrently operating resources.
\end_layout

\begin_layout Enumerate
For safety, physically remove the storage network cables on 
\emph on
all
\emph default
 your cluster nodes.
 Note: the same disclaimer holds.
 MARS really does its best, even when 
\family typewriter
delete-resource
\family default
 is given while the network is fully active and multiple split-brain primaries
 are actively using their local device in parallel (approved by some testcases
 from the automatic test suite, but note that it is impossible to catch
 all possible failure scenarios).
 Don't challenge your fate if you are desperate! Don't 
\emph on
rely
\emph default
 on this! Nothing is absolutely fail-safe!
\end_layout

\begin_layout Enumerate

\series bold
Manually
\series default
 check which surviving disk is usable, and which is the 
\begin_inset Quotes eld
\end_inset

best
\begin_inset Quotes erd
\end_inset

 one for your purpose.
\end_layout

\begin_layout Enumerate
Do 
\family typewriter
modprobe mars
\family default
 
\emph on
only
\emph default
 on that node.
 If that fails, 
\family typewriter
rmmod
\family default
 and/or reboot again, and start over with a completely fresh 
\family typewriter
/mars/
\family default
 partition (
\family typewriter
mkfs.ext4 /mars/
\family default
 or similar) 
\emph on
everywhere
\emph default
 on 
\emph on
all
\emph default
 cluster nodes, and continue with step 7.
\end_layout

\begin_layout Enumerate
If your old 
\family typewriter
/mars/
\family default
 works, and you did not already (forcefully) switch your designated primary
 to the final destination, do it now (see description in section 
\begin_inset CommandInset ref
LatexCommand ref
reference "subsec:Forced-Switching"

\end_inset

).
 Wait until any old logfile data has been replayed.
\end_layout

\begin_layout Enumerate
Say 
\family typewriter
marsadm delete-resource mydata --force
\family default
.
 This will cleanup all internal symlink tree information for the resource,
 but will leave your disk data intact.
\end_layout

\begin_layout Enumerate
Locally build up the new resource(s) as usual, out of the underlying disks.
\end_layout

\begin_layout Enumerate
Check whether the new resource(s) work in standalone mode.
\end_layout

\begin_layout Enumerate
When necessary, repeat these steps with other resources.
\end_layout

\begin_layout Standard
Now you can choose how the rebuild your cluster.
 If you rebuilt 
\family typewriter
/mars/
\family default
 anywhere, you 
\emph on
must
\emph default
 rebuild it on 
\emph on
all
\emph default
 new cluster nodes and start over with a fresh 
\family typewriter
join-cluster
\family default
 on each of them, from scratch.
 It is not possible to mix the old cluster with the new one.
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{enumerate}
\backslash
setcounter{enumi}{9}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
item 
\end_layout

\end_inset

 Finally, do all the necessary 
\family typewriter
join-resource
\family default
s on the respective cluster nodes, according to your new redundancy scenario
 after the failures (e.g.
 after activating spare nodes, etc).
 If you have 
\begin_inset Formula $k>2$
\end_inset

 replicas, start 
\family typewriter
join-resource
\family default
 on the worst / most damaged version first, and start the next preferably
 only after the previous sync has completed successfully.
 This way, you will be permanently retaining some (old and outdated, but
 hopefully potentially usable) replicas while a sync is running.
 Don't start too many syncs in parallel.
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
end{enumerate}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Never use 
\family typewriter
delete-resource
\family default
 twice on the same resource name, after you have already a working standalone
 primary
\begin_inset Foot
status open

\begin_layout Plain Layout
Of course, when you don't have created the 
\emph on
same
\emph default
 resource anew, you may repeat 
\family typewriter
delete-resource
\family default
 on other cluster nodes in order to get rid of local files / symlinks which
 had not been propagated to other nodes before.
\end_layout

\end_inset

.
 You might accidentally destroy your again-working copy! You 
\emph on
can
\emph default
 issue 
\family typewriter
delete-resource
\family default
 multiple times on different nodes, e.g.
 when the network has problems, but doing so 
\emph on
after
\emph default
 re-establishment of the initial primary bears some risk.
 Therefore, the safest way is first deleting the resources everywhere, and
 then starting over afresh.
\end_layout

\begin_layout Standard
Before re-connecting any network cable on any non-primary (new secondaries),
 ensure that all 
\family typewriter
/dev/mars/mydata
\family default
 devices are no longer in use (e.g.
 from an old primary role before the incident happened), and that each local
 disk is detached.
 Only after that, you should be able to safely re-connect the network.
 The 
\family typewriter
delete-resource
\family default
 given at the new primary should propagate now to each of your secondaries,
 and your local disk should be usable for a re-
\family typewriter
join-resource
\family default
.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

When you did not rebuild your cluster from scratch with fresh 
\family typewriter
/mars/
\family default
 filesystems, and one of the old cluster nodes is supposed to be removed
 permanently, use 
\family typewriter
leave-resource
\family default
 (optionally with 
\family typewriter
--host=
\family default
 and/or 
\family typewriter
--force
\family default
) and finally 
\family typewriter
leave-cluster
\family default
.
\end_layout

\begin_layout Chapter
Experts only: Special Trick Switching and Rebuild
\begin_inset CommandInset label
LatexCommand label
name "chap:Experts-only:-Special"

\end_inset


\end_layout

\begin_layout Standard
The following is a further alternative for 
\series bold
experts
\series default
 who really know what they are doing.
 The method is very simple and therefore well-suited for coping with mass
 failures, e.g.
 
\series bold
power blackout of whole datacenters
\series default
.
\end_layout

\begin_layout Standard
In case a primary datacenter fails as a whole for whatever reason and you
 have a backup datacenter, do the following steps in the backup datacenter:
\end_layout

\begin_layout Enumerate
Fencing step: by means of firewalling, 
\series bold
ensure
\series default
 that the (virtually) damaged datacenter nodes 
\series bold
cannot
\series default
 be reached over the network.
 For example, you may place REJECT rules into all of your local iptables
 firewalls at the backup datacenter.
 Alternatively / additionally, you may block the routes at the appropriate
 central router(s) in your network.
\end_layout

\begin_layout Enumerate
Run the sequence 
\family typewriter
marsadm disconnect all; marsadm primary --force all
\family default
 on all nodes in the backup datacenter.
\end_layout

\begin_layout Enumerate
Restart your services in the backup datacenter (as far as necessary).
 Depending on your network setup, further steps like switching BGP routes
 etc may be necessary.
\end_layout

\begin_layout Enumerate
Check that 
\emph on
all
\emph default
 your services are 
\emph on
really
\emph default
 up and running, before you try to repair anything! Failing to do so may
 result in data loss when you execute the following restore method for 
\emph on
experts
\emph default
.
\end_layout

\begin_layout Standard
Now your backup datacenter should continue servicing your clients.
 The final reconstruction of the originally primary datacenter works as
 follows:
\end_layout

\begin_layout Enumerate
At the damaged primary datacenter, ensure that nowhere the MARS kernel module
 is running.
 In case of a power blackout, you shouldn't have executed an automatic 
\family typewriter
modprobe mars
\family default
 anywhere during reboot, so you should be already done when all your nodes
 are up again.
 In case some nodes had no reboot, execute 
\family typewriter
rmmod mars
\family default
 everywhere.
 If 
\family typewriter
rmmod
\family default
 refuses to run, you may need to umount the 
\family typewriter
/dev/mars/mydata
\family default
 device first.
 When nothing else helps, you may just mass reboot your hanging nodes.
\end_layout

\begin_layout Enumerate
At the failed side, do 
\family typewriter
rm -rf /mars/resource-$mydata/
\family default
 for all those resources which had been primary before the blackout.
 Do this 
\emph on
only
\emph default
 for those cases, otherwise you will need unnecessary 
\family typewriter
leave-resource
\family default
s or 
\family typewriter
invalidate
\family default
s later (e.g.
 when half of your nodes were already running at the surving side).
 In order to avoid unnecessary traffic, please do this only as far as really
 necessary.
 Don't remove any other directories.
 In particular, 
\family typewriter
/mars/ips/
\family default
 
\emph on
must
\emph default
 remain intact.
 In case you accidentally deleted them, or you had to re-create 
\family typewriter
/mars/
\family default
 from scratch, try 
\family typewriter
rsync
\family default
 with the correct options.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Caution! before doing this, check that the corresponding directory exists
 at the backup datacenter, and that it is 
\emph on
really
\emph default
 healthy!
\end_layout

\begin_layout Enumerate
Un-Fencing: restore your network firewall / routes and check that they work
 (
\family typewriter
ping
\family default
 etc).
\end_layout

\begin_layout Enumerate
Do 
\family typewriter
modprobe mars
\family default
 everywhere.
 All missing directories and their missing symlinks should be automatically
 fetched from the backup datacenter.
\end_layout

\begin_layout Enumerate
Run 
\family typewriter
marsadm join-resource $res
\family default
, but only at those places where the directory was removed previously, while
 using the same disk devices as before.
 This will minimize actual traffic thanks to the fast full sync algorithm.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

It is 
\series bold
crucial
\series default
 that the fencing step 
\series bold
must
\series default
 be executed 
\emph on
before
\emph default
 any 
\family typewriter
primary --force
\family default
! This way, no split brain will be 
\emph on
visible
\emph default
 at the backup datacenter side, because there is simply no chance for transferri
ng different versions over the network.
 It is also crucial to remove any (potentially diverging) resource directories
 
\emph on
before
\emph default
 the 
\family typewriter
modprobe
\family default
! This way, the backup datacenter never runs into split brain.
 This saves you a lot of detail work for split brain resolution when you
 have to restore bulks of nodes in a short time.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

In case the repair of a full datacenter should take so extremely long that
 some 
\family typewriter
/mars/
\family default
 partitions are about to run out of space at the surviving side, you may
 use the 
\family typewriter
leave-resource --host=failed-node
\family default
 trick described earlier, followed by 
\family typewriter
log-delete-all
\family default
.
 Best if you have prepared a fully automatic script long before the incident,
 which executes suchalike only as far as necessary in each individual case.
\end_layout

\begin_layout Standard
\noindent
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Even better: train such scenarios in advance, and prepare scripts for mass
 automation.
 Look into section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Scripting-HOWTO"

\end_inset

.
\end_layout

\begin_layout Chapter
Creating Backups via Pseudo Snapshots
\begin_inset CommandInset label
LatexCommand label
name "chap:Creating-Backups-via"

\end_inset


\end_layout

\begin_layout Standard
When all your secondaries are all homogenously located in a standby datacenter,
 they will be almost idle all the time.
 This is a waste of computing resources.
\end_layout

\begin_layout Standard
Since MARS is no substitute for a full-fledged backup system, and since
 backups may put high system load onto your active side, you may want to
 utilize your passive hardware resources in a better way.
\end_layout

\begin_layout Standard
MARS supports this thanks to its ability to switch the 
\family typewriter
pause-replay
\family default
 
\emph on
independently
\emph default
 from 
\family typewriter
pause-fetch
\family default
.
\end_layout

\begin_layout Standard
The basic idea is simple: just use 
\family typewriter
pause-replay
\family default
 at your secondary site, but leave the replication of transaction logfiles
 intact by deliberately 
\emph on
not
\emph default
 saying 
\family typewriter
pause-fetch
\family default
.
 This way, your secondary replica (block device) will stay frozen for a
 limited time, without loosing your redundancy: since the transaction logs
 will continue to replicate in the meantime, you can start 
\family typewriter
resume-replay
\family default
 at any time, in particular when a primary-side incident should happen unexpecte
dly.
 The former secondary will just catch up by replaying the outstanding parts
 of the transaction logs in order to become recent.
\end_layout

\begin_layout Standard
However, some 
\emph on
details
\emph default
 have to be obeyed.
 In particular, the current version of MARS needs an additional 
\family typewriter
detach
\family default
 operation, in order to release exclusive access to the underlying disk
 
\family typewriter
/dev/lv/$res
\family default
.
 Future versions of MARS are planned to support this more directly, without
 need for an intermediate 
\family typewriter
detach
\family default
 operation.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Beware: 
\family typewriter
mount -o ro /dev/vg/$res
\family default
 can lead to 
\series bold
unnoticed write operations
\series default
 if you are not careful! Some journalling filesystems like 
\family typewriter
xfs
\family default
 or 
\family typewriter
ext4
\family default
 may replay their journals onto the disk, leading to 
\emph on
binary
\emph default
 differences and thus 
\series bold
destroying your consistency
\series default
 later when you re-enable 
\family typewriter
resume-replay
\family default
!
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 9
	scale 5

\end_inset

Therefore, you may use small LVM snapshots (only in such cases).
 Typically, 
\family typewriter
xfs
\family default
 journal replay will require only a few megabytes.
 Therefore you typically don't need much temporary space for this.
 Here is a more detailed description of steps:
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm pause-replay $res
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm detach $res
\end_layout

\begin_layout Enumerate

\family typewriter
lvcreate --size 100m --snapshot --name ro-$res /dev/vg/$res
\end_layout

\begin_layout Enumerate

\family typewriter
mount -o ro /dev/vg/ro-$res /mnt/tmp
\end_layout

\begin_layout Enumerate
Now draw your backup from 
\family typewriter
/mnt/tmp/
\end_layout

\begin_layout Enumerate

\family typewriter
umount /mnt/tmp
\end_layout

\begin_layout Enumerate

\family typewriter
lvremove -f /dev/vg/ro-$res
\end_layout

\begin_layout Enumerate

\family typewriter
marsadm up $res
\end_layout

\begin_layout Standard
Hint: during the backup, the transaction logs will accumulate on 
\family typewriter
/mars/
\family default
.
 In order to avoid overflow of 
\family typewriter
/mars/
\family default
 (c.f.
 section 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Defending-Overflow"

\end_inset

), don't unnecessarily prolong the backup duration.
\end_layout

\begin_layout Chapter
Command Documentation for Userspace Tools
\begin_inset CommandInset label
LatexCommand label
name "chap:Command-Documentation-for"

\end_inset


\end_layout

\begin_layout Section

\family typewriter
marsadm --help
\begin_inset CommandInset label
LatexCommand label
name "sec:marsadm-–help"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
input{marsadm.help}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand input
preview true
filename "common-back-matter.lyx"

\end_inset


\end_layout

\end_body
\end_document