mars/docu/football-user-manual.lyx

#LyX 2.3 created this file. For more info see http://www.lyx.org/
\lyxformat 544
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass scrreprt
\begin_preamble
\usepackage{listings}
\end_preamble
\options abstracton,dvipsnames
\use_default_options true
\begin_modules
customHeadersFooters
enumitem
fixltx2e
\end_modules
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman "default" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family rmdefault
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\use_microtype false
\use_dash_ligatures false
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize 10
\spacing single
\use_hyperref true
\pdf_title "Football User Manual"
\pdf_author "Thomas Schöbel-Theuer"
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen true
\pdf_bookmarksopenlevel 2
\pdf_breaklinks true
\pdf_pdfborder true
\pdf_colorlinks true
\pdf_backref section
\pdf_pdfusetitle true
\papersize a4paper
\use_geometry true
\use_package amsmath 1
\use_package amssymb 1
\use_package cancel 1
\use_package esint 1
\use_package mathdots 1
\use_package mathtools 1
\use_package mhchem 1
\use_package stackrel 1
\use_package stmaryrd 1
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\biblio_style plain
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\justification true
\use_refstyle 1
\use_minted 0
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 3.7cm
\topmargin 2.7cm
\rightmargin 2.8cm
\bottommargin 2.3cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\is_math_indent 0
\math_numbering_side default
\quotes_style english
\dynamic_quotes 0
\papercolumns 1
\papersides 2
\paperpagestyle headings
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
title{Football User Manual}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset include
LatexCommand input
preview true
filename "common-front-matter.lyx"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset CommandInset toc
LatexCommand tableofcontents

\end_inset


\end_layout

\begin_layout Chapter
LV Football / VM Football / Container Football
\begin_inset CommandInset label
LatexCommand label
name "chap:LV-Football"

\end_inset


\end_layout

\begin_layout Standard
The Football scripts can be obtained in two different ways:
\end_layout

\begin_layout Enumerate

\family typewriter
git clone --recurse-submodules https://github.com/schoebel/mars
\begin_inset Newline newline
\end_inset


\family default
then 
\family typewriter
cd mars/football/
\end_layout

\begin_layout Enumerate

\family typewriter
git clone https://github.com/schoebel/football
\end_layout

\begin_layout Standard
The 
\family typewriter
--recurse-submodule
\family default
 method is the preferred way for non-developers because the main repo contains
 a link to the right version of Football.
\end_layout

\begin_layout Standard
When switching branches, you should use 
\family typewriter
git submodule update
\family default
 for synchronizing the Football submodule with the MARS main checkout.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset

Recommended MARS branch for playing Football is 
\family typewriter
mars0.1a.y
\family default
.
 Although the old stable branch 
\family typewriter
mars0.1.y
\family default
 has been updated for the most important 
\family typewriter
marsadm
\family default
 features 
\family typewriter
merge-cluster
\family default
 and 
\family typewriter
split-cluster
\family default
, it does not scale well for Football and can cause operational problems
 when merging too many hosts together, showing some 
\begin_inset Formula $O(n^{2})$
\end_inset

 metadata update behaviour where 
\begin_inset Formula $n$
\end_inset

 is the number of machines in a MARS cluster.
 The future branch 
\family typewriter
mars0.1b.y
\family default
 will contain more scalability improvements; in particular the 
\family typewriter
split-cluster
\family default
 operation should no longer be needed at all because it is planned to scale
 with 
\begin_inset Formula $O(k)$
\end_inset

 where 
\begin_inset Formula $k$
\end_inset

 is the number of resources at a 
\emph on
single
\emph default
 host.
 This should allow creation of a 
\emph on
virtual(!)
\emph default
 
\family typewriter
BigCluster
\family default
 pool at 
\emph on
metadata
\emph default
 level (where metadata transfer rates are typically measured in KiB/s),
 consisting of thousands of machines, while at the same time creating a
 
\family typewriter
LocalSharding
\family default
 or 
\family typewriter
FlexibleSharding
\family default
 model at the realtime IO paths (where some petabytes are pumped through
 thick pipelines).
 Please check the other branches regularly at the github repo whether some
 newer branches will be marked 
\begin_inset Quotes eld
\end_inset

stable
\begin_inset Quotes erd
\end_inset

, or at least 
\begin_inset Quotes eld
\end_inset

beta
\begin_inset Quotes erd
\end_inset

.
 At the moment (spring 2018), 
\family typewriter
mars0.1a.y
\family default
 is marked 
\begin_inset Quotes eld
\end_inset

beta
\begin_inset Quotes erd
\end_inset

 although it is in production at several thousands of machines for several
 months.
\end_layout

\begin_layout Standard
Low-level documentation is available by calling any of the scripts with
 
\family typewriter
--help
\family default
 parameter (see also appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:football-–help"

\end_inset

 ff):
\end_layout

\begin_layout Itemize

\family typewriter
./football.sh --help
\end_layout

\begin_layout Itemize

\family typewriter
./screener.sh --help
\end_layout

\begin_layout Standard
By adding 
\family typewriter
--verbose
\family default
, you can get a list of parameters for configuring and tweaking.
\end_layout

\begin_layout Section
Football Overview
\begin_inset CommandInset label
LatexCommand label
name "sec:Football-Overview"

\end_inset


\end_layout

\begin_layout Standard
Topmost architectural level (not yet implemented):
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/pool-optimizer.fig
	width 100col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
The planned heart of the Football system is the generic pool optimizer,
 which aims to provide a similar functionality than Kubernetes, but working
 on a sharding architecture.
 Instead of controlling 
\emph on
stateless
\emph default
 Docker containers, its designated goal is to control masses of LVs on thousands
 of machines, creating a 
\begin_inset Quotes eld
\end_inset

Virtually Distributed LVM pool
\begin_inset Quotes erd
\end_inset

 (petabytes of total storage), and doing similar things than Software Defined
 Storage (SDS) on the virtual pool.
\end_layout

\begin_layout Standard
In addition to load balancing of storage space (and its special cases like
 hardware lifecycle), there will be designated plugins for dealing with
 CPU and RAM dimensions.
 Further dimensions and a variety of goal functions could be added via future
 plugins.
 The optimizer itself aims to be as generic as possible, while functionality
 and interfaces can be added via plugins and/or drivers.
 Future versions might even support DRBD in addition to MARS.
 The first version may use a simple greedy algorithm for solving the underlying
 
\begin_inset Formula ${\cal NP}$
\end_inset

-complete problem, but could be augmented with more sophisticated problem
 solvers in future.
\end_layout

\begin_layout Standard
The automatic operations generated by pool-optimizer will be customizable
 by dozens of parameters, and also extendable by action plugins.
 At the moment, the following 
\family typewriter
football.sh
\family default
 actions are planned:
\end_layout

\begin_layout Description

\family typewriter
migrate
\family default
 This will move an LV (together with its VM / LXC container / etc) to a
 different machine in the machine pool.
 This is the classical Football 
\begin_inset Quotes eld
\end_inset

kick
\begin_inset Quotes erd
\end_inset

 operation.
\end_layout

\begin_layout Description

\family typewriter
shrink
\family default
 This decreases the occupied LV space of a filesystem (currently only 
\family typewriter
xfs
\family default
 implemented, but easily extendable) via creation of a smaller temporary
 LV at the hypervisor, then transferring all data during operations via
 local 
\family typewriter
rsync
\family default
, then shutting down the VM for a short period, doing a final incremental
 
\family typewriter
rsync
\family default
, renaming the copied temporary LV to its original name, restarting the
 VM on the new version (which contains the same data as before but wastes
 less space), and finally re-establishing the MARS replicas (but of course
 with smaller LV size).
\end_layout

\begin_layout Description

\family typewriter
extend
\family default
 This is much easier than shrinking: it first increases the underlying LV
 size dynamically on all replicas, then 
\family typewriter
marsadm resize
\family default
, and finally calls 
\family typewriter
xfs_growfs
\family default
 while the filesystem remains mounted and while the VM / container is running.
\end_layout

\begin_layout Description

\family typewriter
migrate+shrink
\family default
 Similar to 
\family typewriter
migrate
\family default
 immediately followed by 
\family typewriter
shrink
\family default
, but produces less network traffic and runs faster.
\end_layout

\begin_layout Description

\family typewriter
migrate+shrink+back
\family default
 Use this when there is not enough local temporary space for shrinking.
 The LV is first migrated to a temporary host, then shrunk, and finally
 migrated back to its original position.
\end_layout

\begin_layout Standard
By running the overall system in an endless loop, a control loop for permanent
 optimization can be established.
 Typical periods are each few days, or once a week.
 In addition, manual triggering is also possible.
\end_layout

\begin_layout Standard
The result of an (incremental) pool-optimizer run is a CSV file, which may
 be automatically forwarded to the execution engine 
\family typewriter
football.sh
\family default
 for 
\emph on
manual
\emph default
 execution, or to 
\family typewriter
screener.sh
\family default
 for mass execution on a common control machine.
 Alternatively, intermediate steps like manual checking, filtering etc may
 be inserted into the processing pipeline.
\end_layout

\begin_layout Standard
The excecution engine 
\family typewriter
football.sh
\family default
 rep.
 its 1&1-internal variant 
\family typewriter
tetris.sh
\family default
 is already in production at 1&1, and already reached more than 300 migrations
 per week.
 Architecture of the execution engine:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
	filename images/football.fig
	width 90col%

\end_inset


\end_layout

\begin_layout Standard
\noindent
The so-called Screener is simply a generic program allowing mass execution
 of arbitrary scripts in background 
\family typewriter
screen
\family default
 sessions.
 This allows masses (several hundreds, possibly thousands) of long-lasting
 processes (hours or days) to run 
\emph on
unattended
\emph default
 in background, while allowing a (larger) group of sysadmins to attach /
 detach to 
\family typewriter
screen
\family default
 sessions at any time for corrective by-hand actions, e.g.
 in case of failures or other problems, or for supervision, etc.
\end_layout

\begin_layout Standard
When Screener is combined with the Football execution engine 
\family typewriter
football.sh
\family default
, more specialized functionality is available (via a variety of plugins):
\end_layout

\begin_layout Itemize
Optional waiting for sysadmin confirmation before some customer downtime
 is initiated.
\end_layout

\begin_layout Itemize
Automatic generation of 
\family typewriter
motd
\family default
 status reporting to other sysadmins.
\end_layout

\begin_layout Itemize
Automatic sending of email alerts or status reports, e.g.
 on errors or critical errors, etc.
 By sending email to SMS gateways, real-time alerting can be configured
 (e.g.
 over the weekend).
\end_layout

\begin_layout Itemize
Generic interfacing to external scripts with configurable parameters, e.g.
 for triggering monitoring systems, feeding external databases, etc.
\end_layout

\begin_layout Standard
Screener can detect and will automatically manage the following states (in
 this example, all state lists are empty):
\end_layout

\begin_layout Standard
\begin_inset listings
inline false
status open

\begin_layout Plain Layout

$common_user> ./screener.sh list
\end_layout

\begin_layout Plain Layout

List of waiting:
\end_layout

\begin_layout Plain Layout

List of delayed:
\end_layout

\begin_layout Plain Layout

List of condition:
\end_layout

\begin_layout Plain Layout

List of running:
\end_layout

\begin_layout Plain Layout

List of critical:
\end_layout

\begin_layout Plain Layout

List of serious:
\end_layout

\begin_layout Plain Layout

List of interrupted:
\end_layout

\begin_layout Plain Layout

List of illegal:
\end_layout

\begin_layout Plain Layout

List of failed:
\end_layout

\begin_layout Plain Layout

List of timeouted:
\end_layout

\begin_layout Plain Layout

List of done:
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\noindent
Screener can discriminate the 
\emph on
seriosity
\emph default
 of errors as follows:
\end_layout

\begin_layout Description

\family typewriter
failed
\family default
 An error occurred 
\emph on
outside
\emph default
 of critical sections, e.g.
 during preparation of LV space etc.
 During ordinary operations, VMs / containers are usually running continuously,
 and there is no customer impact to be expected.
 Typically, 
\family typewriter
./screener.sh restart $resource
\family default
 should fix the problem if it is only a temporary problem.
 However, for maximum safety, manual inspection via .
\family typewriter
/screener.sh attach $resource
\family default
 or inspection of the logfile via .
\family typewriter
/screener.sh show $resource
\family default
 is recommended before trying an automatic restart.
\end_layout

\begin_layout Description

\family typewriter
serious
\family default
 An error occured while a VM / container was temporarily stopped, which
 
\series bold
would
\series default
 normally lead to customer downtime, but Football was able to 
\emph on
compensate
\emph default
 the problem 
\emph on
for now
\emph default
 by 
\emph on
automatically
\emph default
 restarting the VM.
 Thus no long-lasting customer impact has likely occurred.
 However, manual inspection and repair by sysadmins is likely necessary.
\end_layout

\begin_layout Description

\family typewriter
critical
\family default
 An 
\emph on
uncompensated
\emph default
 error occured during customer downtime.
 The VM / container is likely down.
 This will need manual sysadmin actions ASAP, such as hardware replacement,
 networking fixes, etc.
\end_layout

\begin_layout Description

\family typewriter
timeouted
\family default
 This means that the script is assumed to hang because it did not produce
 any output for more than 
\family typewriter
$session_timeout
\family default
 seconds (default 3600 * 3 = 3 hours).
\end_layout

\begin_layout Description

\family typewriter
illegal
\family default
 This means that a precondition is not met.
 For example, there is not enough space at the target LVM.
\end_layout

\begin_layout Description

\family typewriter
interrupted
\family default
 Somebody has pressed 
\family typewriter
Ctl-c
\family default
 in a 
\family typewriter
screen
\family default
 session, or has otherwise sent a signal to the running script.
 As a result, a signal 
\family typewriter
trap
\family default
 has been executed.
\end_layout

\begin_layout Standard
\noindent
Ordinary Screener states during execution:
\end_layout

\begin_layout Description

\family typewriter
running
\family default
 This means that a (background) process is currently running.
 You can attach to the screen session either manually via 
\family typewriter
screen -x $pid.$resource
\family default
, or more comfortably via 
\family typewriter
./screener.sh attach $resource
\family default
.
 Then you can use 
\family typewriter
screen
\family default
 as documented in 
\family typewriter
man screen
\family default
.
 The most important operation is detaching via keystrokes 
\family typewriter
Ctrl-a d
\family default
.
 
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

Notice: don't press 
\family typewriter
Ctrl-c
\family default
 unless you know what you are doing.
 In most cases, this will terminate the running process, and in consequence
 lead to 
\family typewriter
\series bold
interrupted
\family default
\series default
 or 
\family typewriter
\series bold
failed
\family default
\series default
 or even 
\family typewriter
\series bold
critical
\family default
\series default
 state (depending on the moment of keypress).
 Depending on parameter 
\family typewriter
drop_shell
\family default
, the Screener session will also terminate, or you will get an interactive
 shell for manual repair.
\end_layout

\begin_layout Description

\family typewriter
waiting
\family default
 When the plugins 
\family typewriter
football-waiting
\family default
 and 
\family typewriter
screener-waiting
\family default
 are configured properly (which is 
\emph on
not
\emph default
 the default), the script execution will pause immediately before a customer
 downtime action would be started.
 Now any sysadmin from the larger group has a chance to 
\family typewriter
./screener attach $resource
\family default
 and to press RETURN to continue the waiting script and to personally watch
 the course of the critical section.
 There are some more comfortable variants like 
\family typewriter
./screener continue $resource
\family default
 for background continuation of a single session, or 
\family typewriter
./screener continue 100
\family default
 which can be used for continuing masses of waiting sessions.
 There are further variants which are automatically attaching to sessions,
 see Appendix 
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:screener–help"

\end_inset

.
\end_layout

\begin_layout Description

\family typewriter
delayed
\family default
 This state is only entered before 
\family typewriter
lvremove $resource
\family default
 is executed (which will destroy your old internal backup copy), and when
 configured appropriately.
 Typically, you also need to configure the 
\family typewriter
$wait_before_cleanup
\family default
 variable in order to avoid endless waiting.
 Notice that old LV data gets soon outdated after a while, so please don't
 unnecessarily prolong the running time of your scripts by choosing too
 long 
\family typewriter
$wait_before_cleanup
\family default
 values.
\end_layout

\begin_layout Description

\family typewriter
condition
\family default
 Special case of delay: some condition is currently not met, such as the
 
\family typewriter
$business_hours
\family default
 feature, where you can configure when customer downtimes are allowed, and
 when not.
\end_layout

\begin_layout Description

\family typewriter
done
\family default
 This means that the script reported successful execution by exit status
 
\family typewriter
0
\family default
.
 The background screen session terminated automatically.
 You can inspect the logfile manually via 
\family typewriter
./screener.sh show $resource
\family default
, or by looking into the directory 
\family typewriter
$screener_logdir/done/
\family default
.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset

Logfiles of other states can also be inspected (or monitored by standard
 tools like 
\family typewriter
grep
\family default
) by looking into sister directories, such as 
\family typewriter
$screener_logdir/running/
\family default
.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

When running Screener for several months or years, old logfiles will accumulate
 in these directories over time.
 Call 
\family typewriter
./screener.sh purge
\family default
 or 
\family typewriter
./screener.sh cron
\family default
 regularly via a cron job, or archieve your old logfiles from time to time
 via another method.
\end_layout

\begin_layout Section
HOWTO instantiate / customize Football
\begin_inset CommandInset label
LatexCommand label
name "sec:HOWTO-instantiate-Football"

\end_inset


\end_layout

\begin_layout Standard
In order to install and operate Football, the recommended 
\emph on
deployment
\emph default
 strategy is bottom-up, layer by layer.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset

Top-down strategies should be used 
\emph on
only
\emph default
, and 
\emph on
only
\emph default
, for planning.
 An Egyptian pyramid can never be built, even if you had some billions of
 workers, by starting at the tip and by creating the foundations as the
 very last step.
 Suchalike attempt would end up in a disaster.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset


\series bold
Testing
\series default
 of each layer 
\series bold
separately
\series default
 is very important.
 Before proceeding to the next higher layer, first ensure that any lower
 layer is working 
\emph on
correctly
\emph default
.
 Otherwise debugging can become tricky.
\end_layout

\begin_layout Subsection
Block Device Layer
\end_layout

\begin_layout Standard
Step-by-step instructions can be found in chapter 
\begin_inset CommandInset ref
LatexCommand vref
reference "chap:Quick-Start-Guide"

\end_inset

.
\end_layout

\begin_layout Standard
Please ensure that your hardware (including RAID controllers and LVM and
 so on), and your operating system, and your network / setup, and MARS is
 working correctly before proceeding to the next layer.
\end_layout

\begin_layout Subsection
Mechanics Layer of Cluster Operations
\begin_inset CommandInset label
LatexCommand label
name "subsec:Mechanics-Layer of Cluster"

\end_inset


\end_layout

\begin_layout Standard
In the following example, it is assumed that 
\family typewriter
systemd
\family default
 is used, as explained in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:systemd-Templates"

\end_inset

, and now applied to 
\family typewriter
vm4711
\family default
 supposed to run on hypervisors 
\family typewriter
hyper1234a
\family default
 (primary role) and 
\family typewriter
hyper1234b
\family default
 (secondary role), which is assumed to be controllable via the following
 
\family typewriter
systemd
\family default
 start and stop units:
\end_layout

\begin_layout Itemize

\family typewriter
marsadm set-systemd-unit vm4711 lxc-vm4711.target vol-vm4711.mount
\end_layout

\begin_layout Standard
Test the cluster mechanics layer like in the following example:
\end_layout

\begin_layout Itemize
On host 
\family typewriter
hyper1234b
\family default
, the following must work: 
\family typewriter
marsadm primary vm4711
\end_layout

\begin_layout Standard
This must result in an automatic handover of 
\family typewriter
vm4711
\family default
 from the current primary site 
\family typewriter
hyper1234a
\family default
 to the new primary 
\family typewriter
hyper1234b
\family default
, as explained in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:systemd-Templates"

\end_inset

.
 Please check that 
\family typewriter
vm4711
\family default
 is running correctly at the new location.
 It must be reachable via network.
 In case you are using BGP because 
\family typewriter
hyper1234a
\family default
 and 
\family typewriter
hyper1234b
\family default
 are located in different datacenters, ensure that BGP is also controlled
 by your 
\family typewriter
systemd
\family default
 unit dependencies, and test it.
\end_layout

\begin_layout Subsection
Mechanics Layer of Football Operations
\begin_inset CommandInset label
LatexCommand label
name "subsec:Mechanics-Layer-of-Football"

\end_inset


\end_layout

\begin_layout Standard
At the moment, there are two alternative plugins already implemented in
 the Football sub-project (see subdirectory 
\family typewriter
football/plugins/
\family default
).
 Of course, you can implement some further plugins.
 Please put them under GPL, and share them.
 Please contact the author of MARS for inclusion into the official MARS
 release.
\end_layout

\begin_layout Description

\family typewriter
football-cm3.sh
\family default
 This plugin can be only used at Shared Hosting Linux (ShaHoLin) at 1&1,
 since it is bound to a specific 
\emph on
proprietary
\emph default
 instance.
 However, the 
\emph on
sourcecode
\emph default
 of the 
\emph on
plugin
\emph default
 itself (not the code called by the plugin, e.g.
 over REST interfaces) is under GPL.
 You can (and 
\emph on
should
\emph default
) 
\emph on
inspect
\emph default
 the plugin code, and 
\series bold
learn
\series default
 how a real-world system (which has grown over some decades and bears a
 lot of history) is actually working at certain points.
\begin_inset Newline newline
\end_inset

This plugin is automatically activated when called via the symlink 
\family typewriter
tetris.sh
\family default
 instead of directly calling 
\family typewriter
football.sh
\family default
.
 This has historic reasons.
\end_layout

\begin_layout Description

\family typewriter
football-basic.sh
\family default
 This plugin uses the new 
\family typewriter
systemd
\family default
 interface of 
\family typewriter
marsadm
\family default
 for controlling the mechanics.
 See section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:systemd-Templates"

\end_inset

.
 You should be familiar with commands like 
\family typewriter
marsadm set-systemd-unit
\family default
.
 Manual handover via 
\family typewriter
marsadm primary $resource
\family default
 must be already working (with high reliability 
\begin_inset Formula $\leadsto$
\end_inset

 check that any 
\family typewriter
umount
\family default
 works everywhere without hangups) before you can start using this plugin
 for 
\family typewriter
football.sh
\family default
.
\begin_inset Newline newline
\end_inset

This plugin is automatically activated when calling football.sh.
 It can be deactivated by overriding variable 
\family typewriter
enable_basic
\family default
=0.
\end_layout

\begin_layout Subsubsection
Configuring and Overriding Variables
\end_layout

\begin_layout Standard
A detailed list of all available customization options can be obtained via
 
\family typewriter
./football.sh --help --verbose
\family default
.
 Each option is documented by some help text, and you can always see the
 default settings.
 See also section 
\begin_inset CommandInset ref
LatexCommand vref
reference "sec:football-help-verbose"

\end_inset

.
\end_layout

\begin_layout Standard
If you create any new plugin for Football, or if you modify an existing
 one, please follow these standards.
 Try to describe any option as concisely as possible.
\end_layout

\begin_layout Standard
Configuring is possible in the following ways, in order of precedence:
\end_layout

\begin_layout Itemize
at the command line via 
\family typewriter
./football.sh --$variable_name=$value $arguments
\family default
.
\end_layout

\begin_layout Itemize
via environment variables, e.g.
 globally via 
\family typewriter
export $variable_name=$value && ./football.sh $arguments
\family default
, or locally via 
\family typewriter
$variable_name=$value ./football.sh $arguments
\family default
.
\end_layout

\begin_layout Itemize
by adding some small 
\family typewriter
football-*.conf
\family default
 files into one of the directories 
\family typewriter
/usr/lib/mars/plugins
\family default
 
\family typewriter
/etc/mars/plugins
\family default
 
\family typewriter
$script_dir/plugins
\family default
 
\family typewriter
$HOME/.mars/plugins
\family default
 
\family typewriter
./plugins
\family default
, in this order of precedence.
 This list of directories can be modifed externally over the environment
 variable 
\family typewriter
football_includes
\family default
 (but not during already running inclusions of 
\family typewriter
football-*.conf
\family default
 files).
\end_layout

\begin_layout Subsubsection

\family typewriter
football-basic.sh
\family default
 Customization
\end_layout

\begin_layout Standard
Here is a brief summary of the most important configuration tasks and options:
\end_layout

\begin_layout Description

\family typewriter
initial_hostname_file
\family default
 Somehow, the 
\family typewriter
football-basic.sh
\family default
 plugin must know the hostnames of your pool.
 Once Football is working, the hostname will be 
\emph on
automatically
\emph default
 maintained whenever 
\family typewriter
marsadm join-cluster
\family default
 or 
\family typewriter
marsadm merge-cluster
\family default
 is executed somewhere.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset

For your hardware deployment strategy, this means the following: just deploy
 any new hardware, or remove your old one (after Football has emptied all
 of your former LV resources).
 It does not matter how you are doing this, e.g.
 via OpenStack, or via the proprietary 
\family typewriter
Schlunix
\family default
 methods used at ShaHoLin, or whatever.
 Then you have the following options for adding the new machines to the
 Football hostname cache (see variable 
\family typewriter
hostname_cache
\family default
):
\end_layout

\begin_deeper
\begin_layout Enumerate
Write the pure hostname(s) into the file as configured with 
\family typewriter
initial_hostname_file
\family default
 (by default: 
\family typewriter
./hostnames.input
\family default
).
 Each hostname must be on its own ASCII line.
 Not only these new hosts will be picked up automatically, but also...
\end_layout

\begin_layout Enumerate
...any further hosts reported anywhere (at the already known hosts) by 
\family typewriter
marsadm view-cluster-members
\family default
, 
\series bold
transitively
\series default
.
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset

Consequence: if you are running the new 
\family typewriter
mars0.1b.y
\family default
 (or newer) branch of MARS, you don't need 
\family typewriter
marsadm split-cluster
\family default
 anymore.
 Then you can operate several thousands of machines as a big 
\series bold
virtual
\series default
 cluster, even if their storage is local (see 
\family typewriter
LocalSharding
\family default
 model described in section 
\begin_inset CommandInset ref
LatexCommand vref
reference "subsec:Variants-of-Sharding"

\end_inset

).
\begin_inset Newline newline
\end_inset


\begin_inset Graphics
	filename images/MatieresCorrosives.png
	lyxscale 50
	scale 17

\end_inset

 Previous versions of MARS, like 
\family typewriter
mars0.1.y
\family default
 and 
\family typewriter
mars0.1a.y
\family default
, are not yet scalable at their 
\series bold
metadata
\series default
 exchange level.
 Trying to 
\family typewriter
join-cluster
\family default
 or 
\family typewriter
merge-cluster
\family default
 several tens or even hundreds of machines with those versions will surely
 lead to a disaster.
 Always use 
\family typewriter
marsadm split-cluster
\family default
 at those versions, regularly.
 First upgrade to the future 
\family typewriter
mars0.1b.y
\family default
 (or later versions) before creating big clusters at 
\emph on
metadata
\emph default
 level!
\end_layout

\begin_layout Enumerate
Use 
\family typewriter
./football.sh basic_add_host $hostname
\family default
 for adding a single new host manually.
 Afterwards, the transitive closure of all reachable hosts is computed as
 usual.
 This may also be used for the very first initialization of a fresh Football
 installation, provided you already have a big cluster at metadata level.
\end_layout

\end_deeper
\begin_layout Standard
Test the Football mechanics like one of the following example command sequences,
 where it is assumed that 
\family typewriter
hyper4321a
\family default
 and 
\family typewriter
hyper4321b
\family default
 are already 
\emph on
newly
\emph default
 deployed hypervisors having enough local LVM storage, and have been already
 added to the MARS cluster via 
\family typewriter
marsadm join-cluster
\family default
, or have been at least added to 
\family typewriter
hostname_cache
\family default
 as explained above:
\end_layout

\begin_layout Itemize

\family typewriter
ssh-add; ./football.sh migrate vm4711 hyper4321a hyper4321b
\end_layout

\begin_layout Itemize

\family typewriter
ssh-add; ./football.sh migrate vm4711 hyper4321a hyper4321b --screener; ./screener.s
h attach vm4711
\end_layout

\begin_layout Standard
Check the automatically produced logfile (via 
\family typewriter
./screener.sh show vm4711
\family default
) that Football has automatically determined the old hypervisor where 
\family typewriter
vm4711
\family default
 was running before, that it has automatically executed 
\family typewriter
marsadm merge-cluster
\family default
 when necessary, and has created the LV replicas at the new hypervisors,
 and has executed some 
\family typewriter
marsadm join-resource
\family default
 commands, has automatically waited for MARS fast fullsync to finish, then
 successfully executed an automatic handover to the new primary hypervisor,
 and finally has destructed the old MARS replicas including their old LVs.
 Check that 
\family typewriter
vm4711
\family default
 is running correctly at the new hypervisor pair, and that handover between
 the new hypervisor sites 
\family typewriter
*a
\family default
 and 
\family typewriter
*b
\family default
 is working correctly.
 
\end_layout

\begin_layout Standard
A larger group of sysadmins can co-work over a central common control machine
 via ssh agent forwarding (which must be enabled in 
\family typewriter
/etc/ssh/sshd_config
\family default
) in the following way:
\end_layout

\begin_layout Itemize
At the workstation: 
\family typewriter
ssh-add; ssh -A football@common-control.mycompany.org
\family default

\begin_inset Newline newline
\end_inset

Then 
\family typewriter
cd $script_dir
\family default
 and run your 
\family typewriter
./football.sh
\family default
 or 
\family typewriter
./screener.sh
\family default
 commands as usual.
 The automatically generated logfiles will be tagged with the 
\emph on
real
\emph default
 usernames from your original workstation login, as reported by 
\family typewriter
ssh-add -l
\family default
, even transitively when using ssh agent forwarding.
 Thus you may use a common username like 
\family typewriter
football
\family default
 on the common
\begin_inset Foot
status open

\begin_layout Plain Layout
Of course, it is also possible to maintain individual accounts for the same
 Unix group, and set 
\family typewriter
umask
\family default
 and common directory permissions accordingly, such that the classical group-wis
e working concept from the 1970s will do the rest.
 This is much more work, but can establish more fine-grained access control.
 Even more sophisticated methods could involve ACLs, but suchalike is probably
 only necessary at extremely high-sensitive installations.
\end_layout

\end_inset

 control machine.
\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename images/lightbulb_brightlit_benj_.png
	lyxscale 12
	scale 7

\end_inset

Hint: use 
\family typewriter
./screener.sh list
\family default
 (or one of its more specific variants like 
\family typewriter
./screener.sh list-running
\family default
) for determining what's currently going on in a larger group of sysadmins.
\end_layout

\begin_layout Chapter
\start_of_appendix
Football Redundancy Diagrams
\begin_inset CommandInset label
LatexCommand label
name "chap:Football-Redundancy-Diagrams"

\end_inset


\end_layout

\begin_layout Standard
The following tables are showing the number of replicas during Football.
 We focus at the common case of starting with 2 replicas, and ending up
 in a total of another 2 replicas at another machines.
 Further cases, involving multiple secondaries, should go analogously.
\end_layout

\begin_layout Standard
Active primaries are colored in red.
\end_layout

\begin_layout Standard
Backup or shadow replicas (which are present at LVM level, but currently
 not used by MARS) are in parentheses.
 In case of emergency, they could be activated again.
\end_layout

\begin_layout Standard
Replicas which are not in parentheses are kept in 
\family typewriter
UpToDate
\family default
 state all the time, until they are retired into backup replicas.
\end_layout

\begin_layout Section
Parallel 
\family typewriter
migrate
\end_layout

\begin_layout Standard
This creates two additional replicas in parallel, at the target pair.
 After handover to the new site, and after some configurable waiting time,
 the old replicas are deleted.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="4" columns="6">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
migrate
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate x 2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Stepwise 
\family typewriter
migrate
\end_layout

\begin_layout Standard
This variant is useful for hardware lifecycle.
 The uplink of the old hardware is only loaded with creation of 1 replica
 in migration step 1.
 Step 2 creates then another replica at the new hardware, which should have
 a better replication network (e.g.
 better uplinks and/or better capacity for cross-traffic between datacenters).
\end_layout

\begin_layout Standard
This variant is selected by parameter 
\family typewriter
migrate_two_phase=1
\family default
.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="5" columns="6">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
2-Step
\begin_inset Newline newline
\end_inset

migrate
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
3
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate 2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Plain 
\family typewriter
shrink
\end_layout

\begin_layout Standard
Here we need to discriminate between replicas with the old size, and the
 new size (which is typically smaller than the old size).
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="5" columns="7">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
shrink
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

old_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

new_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\color inherit
+ (1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Working
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) +
\color red
 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(2)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Finished
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 
\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(2)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Full 
\family typewriter
migrate+shrink
\end_layout

\begin_layout Standard
This variant is 
\emph on
almost
\emph default
 equivalent to 
\family typewriter
migrate
\family default
 followed by 
\family typewriter
shrink
\family default
.
 The only difference is that cleanup is done 
\emph on
later
\emph default
.
 This means, more replicas are kept for some longer time.
 Thus this variant is more safe than doing 
\family typewriter
migrate
\family default
 and 
\family typewriter
shrink
\family default
 separately.
\end_layout

\begin_layout Standard
This variant is selected by parameters 
\family typewriter
migrate_two_phase=0
\family default
 and 
\family typewriter
migrate_always_all=1
\family default
 and 
\family typewriter
migrate_early_cleanup=0
\family default
:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="7" columns="7">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
FULL
\begin_inset Newline newline
\end_inset

migrate+shrink
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

old_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

new_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate x 2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\color inherit
+ (1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Working
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) +
\color red
 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(4)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Finished
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 
\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(4)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
\noindent
There is a variant which does early cleanup, which is roughly equivalent
 to doing a standalone 
\family typewriter
migrate
\family default
 followed by a standalone 
\family typewriter
shrink
\family default
.
 
\end_layout

\begin_layout Standard
This variant is selected by parameters 
\family typewriter
migrate_two_phase=0
\family default
 and 
\family typewriter
migrate_always_all=1
\family default
 and 
\family typewriter
migrate_early_cleanup=1
\family default
.
 It is less safe because it keeps less replicas and is thus less recommended:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="8" columns="7">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
Sequential
\begin_inset Newline newline
\end_inset

migrate+shrink
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

old_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

new_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate x 2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Early Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\color inherit
+ (1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Working
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) +
\color red
 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(2)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Finished
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 
\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(2)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
Stepwise 
\family typewriter
migrate+shrink
\end_layout

\begin_layout Standard
This variant is useful for hardware lifecycle.
 The uplink of the old hardware is only loaded with creation of 1 replica
 in migration step 1.
 Step 2 creates then another replica at the new hardware, which should have
 a better replication network.
 
\end_layout

\begin_layout Standard
This variant is selected by parameters 
\family typewriter
migrate_two_phase=1
\family default
 and 
\family typewriter
migrate_always_all=1
\family default
 and 
\family typewriter
migrate_early_cleanup=0
\family default
:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="8" columns="7">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
2-Step
\begin_inset Newline newline
\end_inset

migrate+shrink
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

old_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

new_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
3
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate 2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\color inherit
+ (1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Working
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) +
\color red
 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(4)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Finished
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 
\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(4)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
\noindent
This variant can also be combined with early cleanup.
 The result is similar to above.
 The only difference is that the second additional replica is created at
 the new hardware.
 
\end_layout

\begin_layout Standard
This variant is selected by parameters 
\family typewriter
migrate_two_phase=1
\family default
 and 
\family typewriter
migrate_always_all=0
\family default
 and 
\family typewriter
migrate_early_cleanup=
\family default
1.
 Again, this variant is less safe and therefore less recommended.
\end_layout

\begin_layout Standard
However, it keeps at least 2 (backup) replicas all the time and thus could
 be an alternative when decommissioning of old hardware is time-critical.
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="9" columns="7">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
2-Step
\begin_inset Newline newline
\end_inset

migrate+shrink
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

old_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

new_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
3
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate 2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
4
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Early Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\color inherit
+ (1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Working
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) +
\color red
 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(2)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Finished
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 
\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(2)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Section
FAST 
\family typewriter
migrate+shrink
\end_layout

\begin_layout Standard
This variant tries to keep a balance by not creating too many unnecessary
 replicas and to reduce network traffic.
\end_layout

\begin_layout Standard
This variant is selected by parameters 
\family typewriter
migrate_two_phase=0
\family default
 and 
\family typewriter
migrate_always_all=0
\family default
 and 
\family typewriter
migrate_early_cleanup=0
\family default
:
\end_layout

\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
<lyxtabular version="3" rows="7" columns="7">
<features tabularvalignment="middle">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<column alignment="center" valignment="top">
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "14col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
FAST
\begin_inset Newline newline
\end_inset

migrate+shrink
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
SRC
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Primary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "10col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
DST
\begin_inset Newline newline
\end_inset

Secondary
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

old_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
\begin_inset Box Frameless
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "12col%"
special "none"
height "1in"
height_special "totalheight"
thickness "0.4pt"
separation "3pt"
shadowsize "4pt"
framecolor "black"
backgroundcolor "none"
status open

\begin_layout Plain Layout
# Replicas
\begin_inset Newline newline
\end_inset

new_size
\end_layout

\end_inset


\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Migrate x 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
3
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Start
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1 
\color inherit
+ (1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
3
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Working
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) +
\color red
 1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(3)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
Shrink Finished
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(1) + 
\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
(3)
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
<row>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
After Cleanup
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout

\color red
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
1
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
-
\end_layout

\end_inset
</cell>
<cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
\begin_inset Text

\begin_layout Plain Layout
2
\end_layout

\end_inset
</cell>
</row>
</lyxtabular>

\end_inset


\end_layout

\begin_layout Standard
\noindent
As before, this could 
\emph on
theoretically
\emph default
 be combined with early cleanup.
 Such a combination is however not recommended because there is one intermediate
 step where all existing replicas are at the DST primary, and thus this
 one machine must not fail.
\end_layout

\begin_layout Chapter
Command Documentation for Userspace Tools
\begin_inset CommandInset label
LatexCommand label
name "chap:Command-Documentation-for"

\end_inset


\end_layout

\begin_layout Section

\family typewriter
football.sh --help
\begin_inset CommandInset label
LatexCommand label
name "sec:football-–help"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
input{football.help}
\end_layout

\end_inset


\end_layout

\begin_layout Section

\family typewriter
football.sh --help --verbose
\begin_inset CommandInset label
LatexCommand label
name "sec:football-help-verbose"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
input{football-verbose.help}
\end_layout

\end_inset


\end_layout

\begin_layout Section

\family typewriter
screener.sh --help
\begin_inset CommandInset label
LatexCommand label
name "sec:screener–help"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
input{screener.help}
\end_layout

\end_inset


\end_layout

\begin_layout Section

\family typewriter
screener.sh --help --verbose
\begin_inset CommandInset label
LatexCommand label
name "sec:screener-help-verbose"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
input{screener-verbose.help}
\end_layout

\end_inset


\end_layout

\begin_layout Chapter
GNU Free Documentation License
\begin_inset CommandInset label
LatexCommand label
name "chap:GNU-FDL"

\end_inset


\end_layout

\begin_layout Standard
\noindent

\family typewriter
\size footnotesize
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
lstinputlisting{fdl.txt}
\end_layout

\end_inset


\end_layout

\end_body
\end_document