#LyX 2.3 created this file. For more info see http://www.lyx.org/ \lyxformat 544 \begin_document \begin_header \save_transient_properties true \origin unavailable \textclass scrreprt \begin_preamble \usepackage{listings} \end_preamble \options abstracton,most,usenames,dvipsnames \use_default_options true \begin_modules customHeadersFooters enumitem fixltx2e tcolorbox \end_modules \maintain_unincluded_children false \language english \language_package default \inputencoding auto \fontencoding global \font_roman "default" "default" \font_sans "default" "default" \font_typewriter "default" "default" \font_math "auto" "auto" \font_default_family rmdefault \use_non_tex_fonts false \font_sc false \font_osf false \font_sf_scale 100 100 \font_tt_scale 100 100 \use_microtype false \use_dash_ligatures false \graphics default \default_output_format default \output_sync 0 \bibtex_command default \index_command default \paperfontsize 10 \spacing single \use_hyperref true \pdf_title "Football User Manual" \pdf_author "Thomas Schöbel-Theuer" \pdf_bookmarks true \pdf_bookmarksnumbered false \pdf_bookmarksopen true \pdf_bookmarksopenlevel 2 \pdf_breaklinks true \pdf_pdfborder true \pdf_colorlinks true \pdf_backref section \pdf_pdfusetitle true \papersize a4paper \use_geometry true \use_package amsmath 1 \use_package amssymb 1 \use_package cancel 1 \use_package esint 1 \use_package mathdots 1 \use_package mathtools 1 \use_package mhchem 1 \use_package stackrel 1 \use_package stmaryrd 1 \use_package undertilde 1 \cite_engine basic \cite_engine_type default \biblio_style plain \use_bibtopic false \use_indices false \paperorientation portrait \suppress_date false \justification true \use_refstyle 1 \use_minted 0 \index Index \shortcut idx \color #008000 \end_index \leftmargin 3.7cm \topmargin 2.7cm \rightmargin 2.8cm \bottommargin 2.3cm \secnumdepth 3 \tocdepth 3 \paragraph_separation indent \paragraph_indentation default \is_math_indent 0 \math_numbering_side default \quotes_style english \dynamic_quotes 0 \papercolumns 1 \papersides 2 \paperpagestyle headings \tracking_changes false \output_changes false \html_math_output 0 \html_css_as_file 0 \html_be_strict false \end_header \begin_body \begin_layout Standard \begin_inset ERT status open \begin_layout Plain Layout \backslash title{Football User Manual} \end_layout \end_inset \end_layout \begin_layout Standard \begin_inset CommandInset include LatexCommand input preview true filename "common-front-matter.lyx" \end_inset \end_layout \begin_layout Standard \begin_inset CommandInset toc LatexCommand tableofcontents \end_inset \end_layout \begin_layout Chapter LV Football / VM Football / Container Football \begin_inset CommandInset label LatexCommand label name "chap:LV-Football" \end_inset \end_layout \begin_layout Standard The Football scripts can be obtained in two different ways: \end_layout \begin_layout Enumerate \family typewriter git clone --recurse-submodules https://github.com/schoebel/mars \begin_inset Newline newline \end_inset \family default then \family typewriter cd mars/football/ \end_layout \begin_layout Enumerate \family typewriter git clone https://github.com/schoebel/football \end_layout \begin_layout Standard The \family typewriter --recurse-submodule \family default method is the preferred way for non-developers because the main repo contains a link to the right version of Football. \end_layout \begin_layout Standard When switching branches, you should use \family typewriter git submodule update \family default for synchronizing the Football submodule with the MARS main checkout. \end_layout \begin_layout Standard \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset Recommended MARS branch for playing Football is \family typewriter mars0.1a.y \family default . Although the old stable branch \family typewriter mars0.1.y \family default has been updated for the most important \family typewriter marsadm \family default features \family typewriter merge-cluster \family default and \family typewriter split-cluster \family default , it does not scale well for Football and can cause operational problems when merging too many hosts together, showing some \begin_inset Formula $O(n^{2})$ \end_inset metadata update behaviour where \begin_inset Formula $n$ \end_inset is the number of machines in a MARS cluster. The future branch \family typewriter mars0.1b.y \family default will contain more scalability improvements; in particular the \family typewriter split-cluster \family default operation should no longer be needed at all because it is planned to scale with \begin_inset Formula $O(k)$ \end_inset where \begin_inset Formula $k$ \end_inset is the number of resources at a \emph on single \emph default host. This should allow creation of a \emph on virtual(!) \emph default \family typewriter BigCluster \family default pool at \emph on metadata \emph default level (where metadata transfer rates are typically measured in KiB/s), consisting of thousands of machines, while at the same time creating a \family typewriter LocalSharding \family default or \family typewriter FlexibleSharding \family default model at the realtime IO paths (where some petabytes are pumped through thick pipelines). Please check the other branches regularly at the github repo whether some newer branches will be marked \begin_inset Quotes eld \end_inset stable \begin_inset Quotes erd \end_inset , or at least \begin_inset Quotes eld \end_inset beta \begin_inset Quotes erd \end_inset . At the moment (spring 2018), \family typewriter mars0.1a.y \family default is marked \begin_inset Quotes eld \end_inset beta \begin_inset Quotes erd \end_inset although it is in production at several thousands of machines for several months. \end_layout \begin_layout Standard Low-level documentation is available by calling any of the scripts with \family typewriter --help \family default parameter (see also appendix \begin_inset CommandInset ref LatexCommand ref reference "sec:football-–help" \end_inset ff): \end_layout \begin_layout Itemize \family typewriter ./football.sh --help \end_layout \begin_layout Itemize \family typewriter ./screener.sh --help \end_layout \begin_layout Standard By adding \family typewriter --verbose \family default , you can get a list of parameters for configuring and tweaking. \end_layout \begin_layout Section Football Overview \begin_inset CommandInset label LatexCommand label name "sec:Football-Overview" \end_inset \end_layout \begin_layout Standard Topmost architectural level (not yet implemented): \end_layout \begin_layout Standard \noindent \align center \begin_inset Graphics filename images/pool-optimizer.fig width 100col% \end_inset \end_layout \begin_layout Standard \noindent The planned heart of the Football system is the generic pool optimizer, which aims to provide a similar functionality than Kubernetes, but working on a sharding architecture. Instead of controlling \emph on stateless \emph default Docker containers, its designated goal is to control masses of LVs on thousands of machines, creating a \begin_inset Quotes eld \end_inset Virtually Distributed LVM pool \begin_inset Quotes erd \end_inset (petabytes of total storage), and doing similar things than Software Defined Storage (SDS) on the virtual pool. \end_layout \begin_layout Standard In addition to load balancing of storage space (and its special cases like hardware lifecycle), there will be designated plugins for dealing with CPU and RAM dimensions. Further dimensions and a variety of goal functions could be added via future plugins. The optimizer itself aims to be as generic as possible, while functionality and interfaces can be added via plugins and/or drivers. Future versions might even support DRBD in addition to MARS. The first version may use a simple greedy algorithm for solving the underlying \begin_inset Formula ${\cal NP}$ \end_inset -complete problem, but could be augmented with more sophisticated problem solvers in future. \end_layout \begin_layout Standard The automatic operations generated by pool-optimizer will be customizable by dozens of parameters, and also extendable by action plugins. At the moment, the following \family typewriter football.sh \family default actions are planned: \end_layout \begin_layout Description \family typewriter migrate \family default This will move an LV (together with its VM / LXC container / etc) to a different machine in the machine pool. This is the classical Football \begin_inset Quotes eld \end_inset kick \begin_inset Quotes erd \end_inset operation. \end_layout \begin_layout Description \family typewriter shrink \family default This decreases the occupied LV space of a filesystem (currently only \family typewriter xfs \family default implemented, but easily extendable) via creation of a smaller temporary LV at the hypervisor, then transferring all data during operations via local \family typewriter rsync \family default , then shutting down the VM for a short period, doing a final incremental \family typewriter rsync \family default , renaming the copied temporary LV to its original name, restarting the VM on the new version (which contains the same data as before but wastes less space), and finally re-establishing the MARS replicas (but of course with smaller LV size). \end_layout \begin_layout Description \family typewriter extend \family default This is much easier than shrinking: it first increases the underlying LV size dynamically on all replicas, then \family typewriter marsadm resize \family default , and finally calls \family typewriter xfs_growfs \family default while the filesystem remains mounted and while the VM / container is running. \end_layout \begin_layout Description \family typewriter migrate+shrink \family default Similar to \family typewriter migrate \family default immediately followed by \family typewriter shrink \family default , but produces less network traffic and runs faster. \end_layout \begin_layout Description \family typewriter migrate+shrink+back \family default Use this when there is not enough local temporary space for shrinking. The LV is first migrated to a temporary host, then shrunk, and finally migrated back to its original position. \end_layout \begin_layout Standard By running the overall system in an endless loop, a control loop for permanent optimization can be established. Typical periods are each few days, or once a week. In addition, manual triggering is also possible. \end_layout \begin_layout Standard The result of an (incremental) pool-optimizer run is a CSV file, which may be automatically forwarded to the execution engine \family typewriter football.sh \family default for \emph on manual \emph default execution, or to \family typewriter screener.sh \family default for mass execution on a common control machine. Alternatively, intermediate steps like manual checking, filtering etc may be inserted into the processing pipeline. \end_layout \begin_layout Standard The excecution engine \family typewriter football.sh \family default rep. its 1&1-internal variant \family typewriter tetris.sh \family default is already in production at 1&1, and already reached more than 300 migrations per week. Architecture of the execution engine: \end_layout \begin_layout Standard \noindent \align center \begin_inset Graphics filename images/football.fig width 90col% \end_inset \end_layout \begin_layout Standard \noindent The so-called Screener is simply a generic program allowing mass execution of arbitrary scripts in background \family typewriter screen \family default sessions. This allows masses (several hundreds, possibly thousands) of long-lasting processes (hours or days) to run \emph on unattended \emph default in background, while allowing a (larger) group of sysadmins to attach / detach to \family typewriter screen \family default sessions at any time for corrective by-hand actions, e.g. in case of failures or other problems, or for supervision, etc. \end_layout \begin_layout Standard When Screener is combined with the Football execution engine \family typewriter football.sh \family default , more specialized functionality is available (via a variety of plugins): \end_layout \begin_layout Itemize Optional waiting for sysadmin confirmation before some customer downtime is initiated. \end_layout \begin_layout Itemize Automatic generation of \family typewriter motd \family default status reporting to other sysadmins. \end_layout \begin_layout Itemize Automatic sending of email alerts or status reports, e.g. on errors or critical errors, etc. By sending email to SMS gateways, real-time alerting can be configured (e.g. over the weekend). \end_layout \begin_layout Itemize Generic interfacing to external scripts with configurable parameters, e.g. for triggering monitoring systems, feeding external databases, etc. \end_layout \begin_layout Standard Screener can detect and will automatically manage the following states (in this example, all state lists are empty): \end_layout \begin_layout Standard \begin_inset listings inline false status open \begin_layout Plain Layout $common_user> ./screener.sh list \end_layout \begin_layout Plain Layout List of waiting: \end_layout \begin_layout Plain Layout List of delayed: \end_layout \begin_layout Plain Layout List of condition: \end_layout \begin_layout Plain Layout List of running: \end_layout \begin_layout Plain Layout List of critical: \end_layout \begin_layout Plain Layout List of serious: \end_layout \begin_layout Plain Layout List of interrupted: \end_layout \begin_layout Plain Layout List of illegal: \end_layout \begin_layout Plain Layout List of failed: \end_layout \begin_layout Plain Layout List of timeouted: \end_layout \begin_layout Plain Layout List of done: \end_layout \end_inset \end_layout \begin_layout Standard \noindent Screener can discriminate the \emph on seriosity \emph default of errors as follows: \end_layout \begin_layout Description \family typewriter failed \family default An error occurred \emph on outside \emph default of critical sections, e.g. during preparation of LV space etc. During ordinary operations, VMs / containers are usually running continuously, and there is no customer impact to be expected. Typically, \family typewriter ./screener.sh restart $resource \family default should fix the problem if it is only a temporary problem. However, for maximum safety, manual inspection via . \family typewriter /screener.sh attach $resource \family default or inspection of the logfile via . \family typewriter /screener.sh show $resource \family default is recommended before trying an automatic restart. \end_layout \begin_layout Description \family typewriter serious \family default An error occured while a VM / container was temporarily stopped, which \series bold would \series default normally lead to customer downtime, but Football was able to \emph on compensate \emph default the problem \emph on for now \emph default by \emph on automatically \emph default restarting the VM. Thus no long-lasting customer impact has likely occurred. However, manual inspection and repair by sysadmins is likely necessary. \end_layout \begin_layout Description \family typewriter critical \family default An \emph on uncompensated \emph default error occured during customer downtime. The VM / container is likely down. This will need manual sysadmin actions ASAP, such as hardware replacement, networking fixes, etc. \end_layout \begin_layout Description \family typewriter timeouted \family default This means that the script is assumed to hang because it did not produce any output for more than \family typewriter $session_timeout \family default seconds (default 3600 * 3 = 3 hours). \end_layout \begin_layout Description \family typewriter illegal \family default This means that a precondition is not met. For example, there is not enough space at the target LVM. \end_layout \begin_layout Description \family typewriter interrupted \family default Somebody has pressed \family typewriter Ctl-c \family default in a \family typewriter screen \family default session, or has otherwise sent a signal to the running script. As a result, a signal \family typewriter trap \family default has been executed. \end_layout \begin_layout Standard \noindent Ordinary Screener states during execution: \end_layout \begin_layout Description \family typewriter running \family default This means that a (background) process is currently running. You can attach to the screen session either manually via \family typewriter screen -x $pid.$resource \family default , or more comfortably via \family typewriter ./screener.sh attach $resource \family default . Then you can use \family typewriter screen \family default as documented in \family typewriter man screen \family default . The most important operation is detaching via keystrokes \family typewriter Ctrl-a d \family default . \begin_inset Newline newline \end_inset \begin_inset Graphics filename images/MatieresCorrosives.png lyxscale 50 scale 17 \end_inset Notice: don't press \family typewriter Ctrl-c \family default unless you know what you are doing. In most cases, this will terminate the running process, and in consequence lead to \family typewriter \series bold interrupted \family default \series default or \family typewriter \series bold failed \family default \series default or even \family typewriter \series bold critical \family default \series default state (depending on the moment of keypress). Depending on parameter \family typewriter drop_shell \family default , the Screener session will also terminate, or you will get an interactive shell for manual repair. \end_layout \begin_layout Description \family typewriter waiting \family default When the plugins \family typewriter football-waiting \family default and \family typewriter screener-waiting \family default are configured properly (which is \emph on not \emph default the default), the script execution will pause immediately before a customer downtime action would be started. Now any sysadmin from the larger group has a chance to \family typewriter ./screener attach $resource \family default and to press RETURN to continue the waiting script and to personally watch the course of the critical section. There are some more comfortable variants like \family typewriter ./screener continue $resource \family default for background continuation of a single session, or \family typewriter ./screener continue 100 \family default which can be used for continuing masses of waiting sessions. There are further variants which are automatically attaching to sessions, see Appendix \begin_inset CommandInset ref LatexCommand ref reference "sec:screener–help" \end_inset . \end_layout \begin_layout Description \family typewriter delayed \family default This state is only entered before \family typewriter lvremove $resource \family default is executed (which will destroy your old internal backup copy), and when configured appropriately. Typically, you also need to configure the \family typewriter $wait_before_cleanup \family default variable in order to avoid endless waiting. Notice that old LV data gets soon outdated after a while, so please don't unnecessarily prolong the running time of your scripts by choosing too long \family typewriter $wait_before_cleanup \family default values. \end_layout \begin_layout Description \family typewriter condition \family default Special case of delay: some condition is currently not met, such as the \family typewriter $business_hours \family default feature, where you can configure when customer downtimes are allowed, and when not. \end_layout \begin_layout Description \family typewriter done \family default This means that the script reported successful execution by exit status \family typewriter 0 \family default . The background screen session terminated automatically. You can inspect the logfile manually via \family typewriter ./screener.sh show $resource \family default , or by looking into the directory \family typewriter $screener_logdir/done/ \family default . \end_layout \begin_layout Standard \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset Logfiles of other states can also be inspected (or monitored by standard tools like \family typewriter grep \family default ) by looking into sister directories, such as \family typewriter $screener_logdir/running/ \family default . \end_layout \begin_layout Standard \begin_inset Graphics filename images/MatieresCorrosives.png lyxscale 50 scale 17 \end_inset When running Screener for several months or years, old logfiles will accumulate in these directories over time. Call \family typewriter ./screener.sh purge \family default or \family typewriter ./screener.sh cron \family default regularly via a cron job, or archieve your old logfiles from time to time via another method. \end_layout \begin_layout Section HOWTO instantiate / customize Football \begin_inset CommandInset label LatexCommand label name "sec:HOWTO-instantiate-Football" \end_inset \end_layout \begin_layout Standard In order to install and operate Football, the recommended \emph on deployment \emph default strategy is bottom-up, layer by layer. \end_layout \begin_layout Standard \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset Top-down strategies should be used \emph on only \emph default , and \emph on only \emph default , for planning. An Egyptian pyramid can never be built, even if you had some billions of workers, by starting at the tip and by creating the foundations as the very last step. Suchalike attempt would end up in a disaster. \end_layout \begin_layout Standard \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset \series bold Testing \series default of each layer \series bold separately \series default is very important. Before proceeding to the next higher layer, first ensure that any lower layer is working \emph on correctly \emph default . Otherwise debugging can become tricky. \end_layout \begin_layout Subsection Block Device Layer \end_layout \begin_layout Standard Step-by-step instructions can be found in chapter \begin_inset CommandInset ref LatexCommand vref reference "chap:Quick-Start-Guide" \end_inset . \end_layout \begin_layout Standard Please ensure that your hardware (including RAID controllers and LVM and so on), and your operating system, and your network / setup, and MARS is working correctly before proceeding to the next layer. \end_layout \begin_layout Subsection Mechanics Layer of Cluster Operations \begin_inset CommandInset label LatexCommand label name "subsec:Mechanics-Layer of Cluster" \end_inset \end_layout \begin_layout Standard In the following example, it is assumed that \family typewriter systemd \family default is used, as explained in section \begin_inset CommandInset ref LatexCommand vref reference "sec:systemd-Templates" \end_inset , and now applied to \family typewriter vm4711 \family default supposed to run on hypervisors \family typewriter hyper1234a \family default (primary role) and \family typewriter hyper1234b \family default (secondary role), which is assumed to be controllable via the following \family typewriter systemd \family default start and stop units: \end_layout \begin_layout Itemize \family typewriter marsadm set-systemd-unit vm4711 lxc-vm4711.target vol-vm4711.mount \end_layout \begin_layout Standard Test the cluster mechanics layer like in the following example: \end_layout \begin_layout Itemize On host \family typewriter hyper1234b \family default , the following must work: \family typewriter marsadm primary vm4711 \end_layout \begin_layout Standard This must result in an automatic handover of \family typewriter vm4711 \family default from the current primary site \family typewriter hyper1234a \family default to the new primary \family typewriter hyper1234b \family default , as explained in section \begin_inset CommandInset ref LatexCommand vref reference "sec:systemd-Templates" \end_inset . Please check that \family typewriter vm4711 \family default is running correctly at the new location. It must be reachable via network. In case you are using BGP because \family typewriter hyper1234a \family default and \family typewriter hyper1234b \family default are located in different datacenters, ensure that BGP is also controlled by your \family typewriter systemd \family default unit dependencies, and test it. \end_layout \begin_layout Subsection Mechanics Layer of Football Operations \begin_inset CommandInset label LatexCommand label name "subsec:Mechanics-Layer-of-Football" \end_inset \end_layout \begin_layout Standard At the moment, there are two alternative plugins already implemented in the Football sub-project (see subdirectory \family typewriter football/plugins/ \family default ). Of course, you can implement some further plugins. Please put them under GPL, and share them. Please contact the author of MARS for inclusion into the official MARS release. \end_layout \begin_layout Description \family typewriter football-cm3.sh \family default This plugin can be only used at Shared Hosting Linux (ShaHoLin) at 1&1, since it is bound to a specific \emph on proprietary \emph default instance. However, the \emph on sourcecode \emph default of the \emph on plugin \emph default itself (not the code called by the plugin, e.g. over REST interfaces) is under GPL. You can (and \emph on should \emph default ) \emph on inspect \emph default the plugin code, and \series bold learn \series default how a real-world system (which has grown over some decades and bears a lot of history) is actually working at certain points. \begin_inset Newline newline \end_inset This plugin is automatically activated when called via the symlink \family typewriter tetris.sh \family default instead of directly calling \family typewriter football.sh \family default . This has historic reasons. \end_layout \begin_layout Description \family typewriter football-basic.sh \family default This plugin uses the new \family typewriter systemd \family default interface of \family typewriter marsadm \family default for controlling the mechanics. See section \begin_inset CommandInset ref LatexCommand vref reference "sec:systemd-Templates" \end_inset . You should be familiar with commands like \family typewriter marsadm set-systemd-unit \family default . Manual handover via \family typewriter marsadm primary $resource \family default must be already working (with high reliability \begin_inset Formula $\leadsto$ \end_inset check that any \family typewriter umount \family default works everywhere without hangups) before you can start using this plugin for \family typewriter football.sh \family default . \begin_inset Newline newline \end_inset This plugin is automatically activated when calling football.sh. It can be deactivated by overriding variable \family typewriter enable_basic \family default =0. \end_layout \begin_layout Subsubsection Configuring and Overriding Variables \end_layout \begin_layout Standard A detailed list of all available customization options can be obtained via \family typewriter ./football.sh --help --verbose \family default . Each option is documented by some help text, and you can always see the default settings. See also section \begin_inset CommandInset ref LatexCommand vref reference "sec:football-help-verbose" \end_inset . \end_layout \begin_layout Standard If you create any new plugin for Football, or if you modify an existing one, please follow these standards. Try to describe any option as concisely as possible. \end_layout \begin_layout Standard Configuring is possible in the following ways, in order of precedence: \end_layout \begin_layout Itemize at the command line via \family typewriter ./football.sh --$variable_name=$value $arguments \family default . \end_layout \begin_layout Itemize via environment variables, e.g. globally via \family typewriter export $variable_name=$value && ./football.sh $arguments \family default , or locally via \family typewriter $variable_name=$value ./football.sh $arguments \family default . \end_layout \begin_layout Itemize by adding some small \family typewriter football-*.conf \family default files into one of the directories \family typewriter /usr/lib/mars/plugins \family default \family typewriter /etc/mars/plugins \family default \family typewriter $script_dir/plugins \family default \family typewriter $HOME/.mars/plugins \family default \family typewriter ./plugins \family default , in this order of precedence. This list of directories can be modifed externally over the environment variable \family typewriter football_includes \family default (but not during already running inclusions of \family typewriter football-*.conf \family default files). \end_layout \begin_layout Subsubsection \family typewriter football-basic.sh \family default Customization \end_layout \begin_layout Standard Here is a brief summary of the most important configuration tasks and options: \end_layout \begin_layout Description \family typewriter initial_hostname_file \family default Somehow, the \family typewriter football-basic.sh \family default plugin must know the hostnames of your pool. Once Football is working, the hostname will be \emph on automatically \emph default maintained whenever \family typewriter marsadm join-cluster \family default or \family typewriter marsadm merge-cluster \family default is executed somewhere. \begin_inset Newline newline \end_inset \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset For your hardware deployment strategy, this means the following: just deploy any new hardware, or remove your old one (after Football has emptied all of your former LV resources). It does not matter how you are doing this, e.g. via OpenStack, or via the proprietary \family typewriter Schlunix \family default methods used at ShaHoLin, or whatever. Then you have the following options for adding the new machines to the Football hostname cache (see variable \family typewriter hostname_cache \family default ): \end_layout \begin_deeper \begin_layout Enumerate Write the pure hostname(s) into the file as configured with \family typewriter initial_hostname_file \family default (by default: \family typewriter ./hostnames.input \family default ). Each hostname must be on its own ASCII line. Not only these new hosts will be picked up automatically, but also... \end_layout \begin_layout Enumerate ...any further hosts reported anywhere (at the already known hosts) by \family typewriter marsadm view-cluster-members \family default , \series bold transitively \series default . \begin_inset Newline newline \end_inset \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset Consequence: if you are running the new \family typewriter mars0.1b.y \family default (or newer) branch of MARS, you don't need \family typewriter marsadm split-cluster \family default anymore. Then you can operate several thousands of machines as a big \series bold virtual \series default cluster, even if their storage is local (see \family typewriter LocalSharding \family default model described in section \begin_inset CommandInset ref LatexCommand vref reference "subsec:Variants-of-Sharding" \end_inset ). \begin_inset Newline newline \end_inset \begin_inset Graphics filename images/MatieresCorrosives.png lyxscale 50 scale 17 \end_inset Previous versions of MARS, like \family typewriter mars0.1.y \family default and \family typewriter mars0.1a.y \family default , are not yet scalable at their \series bold metadata \series default exchange level. Trying to \family typewriter join-cluster \family default or \family typewriter merge-cluster \family default several tens or even hundreds of machines with those versions will surely lead to a disaster. Always use \family typewriter marsadm split-cluster \family default at those versions, regularly. First upgrade to the future \family typewriter mars0.1b.y \family default (or later versions) before creating big clusters at \emph on metadata \emph default level! \end_layout \begin_layout Enumerate Use \family typewriter ./football.sh basic_add_host $hostname \family default for adding a single new host manually. Afterwards, the transitive closure of all reachable hosts is computed as usual. This may also be used for the very first initialization of a fresh Football installation, provided you already have a big cluster at metadata level. \end_layout \end_deeper \begin_layout Standard Test the Football mechanics like one of the following example command sequences, where it is assumed that \family typewriter hyper4321a \family default and \family typewriter hyper4321b \family default are already \emph on newly \emph default deployed hypervisors having enough local LVM storage, and have been already added to the MARS cluster via \family typewriter marsadm join-cluster \family default , or have been at least added to \family typewriter hostname_cache \family default as explained above: \end_layout \begin_layout Itemize \family typewriter ssh-add; ./football.sh migrate vm4711 hyper4321a hyper4321b \end_layout \begin_layout Itemize \family typewriter ssh-add; ./football.sh migrate vm4711 hyper4321a hyper4321b --screener; ./screener.s h attach vm4711 \end_layout \begin_layout Standard Check the automatically produced logfile (via \family typewriter ./screener.sh show vm4711 \family default ) that Football has automatically determined the old hypervisor where \family typewriter vm4711 \family default was running before, that it has automatically executed \family typewriter marsadm merge-cluster \family default when necessary, and has created the LV replicas at the new hypervisors, and has executed some \family typewriter marsadm join-resource \family default commands, has automatically waited for MARS fast fullsync to finish, then successfully executed an automatic handover to the new primary hypervisor, and finally has destructed the old MARS replicas including their old LVs. Check that \family typewriter vm4711 \family default is running correctly at the new hypervisor pair, and that handover between the new hypervisor sites \family typewriter *a \family default and \family typewriter *b \family default is working correctly. \end_layout \begin_layout Standard A larger group of sysadmins can co-work over a central common control machine via ssh agent forwarding (which must be enabled in \family typewriter /etc/ssh/sshd_config \family default ) in the following way: \end_layout \begin_layout Itemize At the workstation: \family typewriter ssh-add; ssh -A football@common-control.mycompany.org \family default \begin_inset Newline newline \end_inset Then \family typewriter cd $script_dir \family default and run your \family typewriter ./football.sh \family default or \family typewriter ./screener.sh \family default commands as usual. The automatically generated logfiles will be tagged with the \emph on real \emph default usernames from your original workstation login, as reported by \family typewriter ssh-add -l \family default , even transitively when using ssh agent forwarding. Thus you may use a common username like \family typewriter football \family default on the common \begin_inset Foot status open \begin_layout Plain Layout Of course, it is also possible to maintain individual accounts for the same Unix group, and set \family typewriter umask \family default and common directory permissions accordingly, such that the classical group-wis e working concept from the 1970s will do the rest. This is much more work, but can establish more fine-grained access control. Even more sophisticated methods could involve ACLs, but suchalike is probably only necessary at extremely high-sensitive installations. \end_layout \end_inset control machine. \end_layout \begin_layout Standard \begin_inset Graphics filename images/lightbulb_brightlit_benj_.png lyxscale 12 scale 7 \end_inset Hint: use \family typewriter ./screener.sh list \family default (or one of its more specific variants like \family typewriter ./screener.sh list-running \family default ) for determining what's currently going on in a larger group of sysadmins. \end_layout \begin_layout Chapter \start_of_appendix Football Redundancy Diagrams \begin_inset CommandInset label LatexCommand label name "chap:Football-Redundancy-Diagrams" \end_inset \end_layout \begin_layout Standard The following tables are showing the number of replicas during Football. We focus at the common case of starting with 2 replicas, and ending up in a total of another 2 replicas at another machines. Further cases, involving multiple secondaries, should go analogously. \end_layout \begin_layout Standard Active primaries are colored in red. \end_layout \begin_layout Standard Backup or shadow replicas (which are present at LVM level, but currently not used by MARS) are in parentheses. In case of emergency, they could be activated again. \end_layout \begin_layout Standard Replicas which are not in parentheses are kept in \family typewriter UpToDate \family default state all the time, until they are retired into backup replicas. \end_layout \begin_layout Section Parallel \family typewriter migrate \end_layout \begin_layout Standard This creates two additional replicas in parallel, at the target pair. After handover to the new site, and after some configurable waiting time, the old replicas are deleted. \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout migrate \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate x 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Section Stepwise \family typewriter migrate \end_layout \begin_layout Standard This variant is useful for hardware lifecycle. The uplink of the old hardware is only loaded with creation of 1 replica in migration step 1. Step 2 creates then another replica at the new hardware, which should have a better replication network (e.g. better uplinks and/or better capacity for cross-traffic between datacenters). \end_layout \begin_layout Standard This variant is selected by parameter \family typewriter migrate_two_phase=1 \family default . \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout 2-Step \begin_inset Newline newline \end_inset migrate \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 3 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Section Plain \family typewriter shrink \end_layout \begin_layout Standard Here we need to discriminate between replicas with the old size, and the new size (which is typically smaller than the old size). \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout shrink \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset old_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset new_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \color inherit + (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Working \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (2) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Finished \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (2) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Section Full \family typewriter migrate+shrink \end_layout \begin_layout Standard This variant is \emph on almost \emph default equivalent to \family typewriter migrate \family default followed by \family typewriter shrink \family default . The only difference is that cleanup is done \emph on later \emph default . This means, more replicas are kept for some longer time. Thus this variant is more safe than doing \family typewriter migrate \family default and \family typewriter shrink \family default separately. \end_layout \begin_layout Standard This variant is selected by parameters \family typewriter migrate_two_phase=0 \family default and \family typewriter migrate_always_all=1 \family default and \family typewriter migrate_early_cleanup=0 \family default : \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout FULL \begin_inset Newline newline \end_inset migrate+shrink \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset old_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset new_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate x 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \color inherit + (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Working \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (4) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Finished \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (4) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Standard \noindent There is a variant which does early cleanup, which is roughly equivalent to doing a standalone \family typewriter migrate \family default followed by a standalone \family typewriter shrink \family default . \end_layout \begin_layout Standard This variant is selected by parameters \family typewriter migrate_two_phase=0 \family default and \family typewriter migrate_always_all=1 \family default and \family typewriter migrate_early_cleanup=1 \family default . It is less safe because it keeps less replicas and is thus less recommended: \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout Sequential \begin_inset Newline newline \end_inset migrate+shrink \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset old_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset new_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate x 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Early Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \color inherit + (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Working \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (2) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Finished \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (2) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Section Stepwise \family typewriter migrate+shrink \end_layout \begin_layout Standard This variant is useful for hardware lifecycle. The uplink of the old hardware is only loaded with creation of 1 replica in migration step 1. Step 2 creates then another replica at the new hardware, which should have a better replication network. \end_layout \begin_layout Standard This variant is selected by parameters \family typewriter migrate_two_phase=1 \family default and \family typewriter migrate_always_all=1 \family default and \family typewriter migrate_early_cleanup=0 \family default : \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout 2-Step \begin_inset Newline newline \end_inset migrate+shrink \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset old_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset new_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 3 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \color inherit + (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Working \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (4) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Finished \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (4) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Standard \noindent This variant can also be combined with early cleanup. The result is similar to above. The only difference is that the second additional replica is created at the new hardware. \end_layout \begin_layout Standard This variant is selected by parameters \family typewriter migrate_two_phase=1 \family default and \family typewriter migrate_always_all=0 \family default and \family typewriter migrate_early_cleanup= \family default 1. Again, this variant is less safe and therefore less recommended. \end_layout \begin_layout Standard However, it keeps at least 2 (backup) replicas all the time and thus could be an alternative when decommissioning of old hardware is time-critical. \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout 2-Step \begin_inset Newline newline \end_inset migrate+shrink \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset old_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset new_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 3 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 4 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Early Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \color inherit + (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Working \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (2) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Finished \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (2) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Section FAST \family typewriter migrate+shrink \end_layout \begin_layout Standard This variant tries to keep a balance by not creating too many unnecessary replicas and to reduce network traffic. \end_layout \begin_layout Standard This variant is selected by parameters \family typewriter migrate_two_phase=0 \family default and \family typewriter migrate_always_all=0 \family default and \family typewriter migrate_early_cleanup=0 \family default : \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "14col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout FAST \begin_inset Newline newline \end_inset migrate+shrink \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout SRC \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Primary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "10col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout DST \begin_inset Newline newline \end_inset Secondary \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset old_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \begin_inset Box Frameless position "t" hor_pos "c" has_inner_box 1 inner_pos "t" use_parbox 0 use_makebox 0 width "12col%" special "none" height "1in" height_special "totalheight" thickness "0.4pt" separation "3pt" shadowsize "4pt" framecolor "black" backgroundcolor "none" status open \begin_layout Plain Layout # Replicas \begin_inset Newline newline \end_inset new_size \end_layout \end_inset \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Migrate x 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 3 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Start \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \color inherit + (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 3 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Working \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (3) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout Shrink Finished \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (1) + \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout (3) \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout After Cleanup \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout \color red 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 1 \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout - \end_layout \end_inset \begin_inset Text \begin_layout Plain Layout 2 \end_layout \end_inset \end_inset \end_layout \begin_layout Standard \noindent As before, this could \emph on theoretically \emph default be combined with early cleanup. Such a combination is however not recommended because there is one intermediate step where all existing replicas are at the DST primary, and thus this one machine must not fail. \end_layout \begin_layout Chapter Command Documentation for Userspace Tools \begin_inset CommandInset label LatexCommand label name "chap:Command-Documentation-for" \end_inset \end_layout \begin_layout Section \family typewriter football.sh --help \begin_inset CommandInset label LatexCommand label name "sec:football-–help" \end_inset \end_layout \begin_layout Standard \begin_inset ERT status open \begin_layout Plain Layout \backslash input{football.help} \end_layout \end_inset \end_layout \begin_layout Section \family typewriter football.sh --help --verbose \begin_inset CommandInset label LatexCommand label name "sec:football-help-verbose" \end_inset \end_layout \begin_layout Standard \begin_inset ERT status open \begin_layout Plain Layout \backslash input{football-verbose.help} \end_layout \end_inset \end_layout \begin_layout Section \family typewriter screener.sh --help \begin_inset CommandInset label LatexCommand label name "sec:screener–help" \end_inset \end_layout \begin_layout Standard \begin_inset ERT status open \begin_layout Plain Layout \backslash input{screener.help} \end_layout \end_inset \end_layout \begin_layout Section \family typewriter screener.sh --help --verbose \begin_inset CommandInset label LatexCommand label name "sec:screener-help-verbose" \end_inset \end_layout \begin_layout Standard \begin_inset ERT status open \begin_layout Plain Layout \backslash input{screener-verbose.help} \end_layout \end_inset \end_layout \begin_layout Standard \begin_inset CommandInset include LatexCommand input preview true filename "common-back-matter.lyx" \end_inset \end_layout \end_body \end_document