diff --git a/ChangeLog b/ChangeLog index c0b2d171..69f42f5c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -296,6 +296,16 @@ Hint: branch 0.1a will get a merge from here, and then get the (except Football related ones) will then go to 0.1b. Finally, when 0.1a is stable, I will close this branch. +mars0.1stable63 + * Minor fix: when compiling for some newer kernels (only there), + schedule() could be called during wait for some condition, + worsening performance unnecessarily. + * Minor improvement: starting join-resource in batches + was slow because each was waiting for cluster communication. + Use a manual "marsadm wait-cluster" before starting batches + of join-resource operations. + * Doc: some clarifications on BigCluster scalability behaviour. + mars0.1stable62 * Minor fix: race between join-resource and log-rotate. * Minor fix: report split brain logfile amount only when diff --git a/docu/mars-manual.lyx b/docu/mars-manual.lyx index 5dbd722b..2269744e 100644 --- a/docu/mars-manual.lyx +++ b/docu/mars-manual.lyx @@ -141,7 +141,7 @@ tst@1und1.de \end_layout \begin_layout Date -Version 0.1a-15 +Version 0.1a-16 \end_layout \begin_layout Lowertitleback @@ -1567,14 +1567,32 @@ Some ZFS advocates believe that ZFS snapshots, which were originally designed \begin_layout Standard The following table tries to explain why geo-redundancy is not as simple to achieve as believed, at least without addition of sophisticated additional - means: + means +\begin_inset Foot +status open + +\begin_layout Plain Layout +ZFS advocates often argue with many features which aren't present at other + filesystem types. + The above table shows some dimensions not dealing with properties of local + filesystems, but with +\emph on +problems / tasks +\emph default + arising in long-distance distributed systems involving masses of enterprise-cri +tical storage. +\end_layout + +\end_inset + +: \end_layout \begin_layout Standard \noindent \align center \begin_inset Tabular - + @@ -1691,6 +1709,44 @@ yes FS-specific \end_layout +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Granularity +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +LVs +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +LVs +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +subvolumes +\end_layout + \end_inset @@ -6106,6 +6162,26 @@ Disk failures are regarded as already solved (e.g. already included in the probability of storage node failures. \end_layout +\begin_layout Itemize +We only look at +\series bold +data replication +\series default + with a redundancy degree of a relatively small +\begin_inset Formula $k$ +\end_inset + +. + CRC methods are not used across storage nodes, but may be present +\emph on +internally +\emph default + at some storage nodes, e.g. + RAID-5 or RAID-6 or similar methods. + Notice that CRC methods generally involve very high overhead, and even + won't work in realtime across long distances (geo-redundancy). +\end_layout + \begin_layout Itemize We restrict ourselves to temporary / \series bold @@ -6147,8 +6223,101 @@ The x axis shows the number of basic storage units \begin_inset Formula $n$ \end_inset -, where one basic storage unit equals to the total disk space provided by - one storage node. + from an +\emph on +application +\emph default + perspective, meaning +\begin_inset Quotes eld +\end_inset + +usable storage +\begin_inset Quotes erd +\end_inset + + or +\begin_inset Quotes eld +\end_inset + +net amount of storage +\begin_inset Quotes erd +\end_inset + +. + For simplicitiy of the model, one basic application storage unit equals + to the total disk space provided by one physical storage node in the special + case of +\begin_inset Formula $k=1$ +\end_inset + + replicas. +\begin_inset Newline newline +\end_inset + + +\begin_inset Graphics + filename images/MatieresCorrosives.png + lyxscale 50 + scale 17 + +\end_inset + + Attention! when increasing the number of replicas +\begin_inset Formula $k$ +\end_inset + +, the total number of storage nodes needs to be +\series bold +increased accordingly +\series default +. + Typically, you will need to deploy +\begin_inset Formula $k\cdot n$ +\end_inset + + physical storage nodes in order to get +\begin_inset Formula $n$ +\end_inset + + net storage units from a user's perspective. +\begin_inset Newline newline +\end_inset + + +\begin_inset Graphics + filename images/MatieresCorrosives.png + lyxscale 50 + scale 17 + +\end_inset + + Attention! +\begin_inset space ~ +\end_inset + + +\begin_inset Formula $k$ +\end_inset + + has a strong influence at the +\series bold +price tag +\series default + of any of the competing architectures. + You cannot assume an +\begin_inset Quotes eld +\end_inset + +infinite amount of money +\begin_inset Quotes erd +\end_inset + +. + Therefore, only relatively small +\begin_inset Formula $k$ +\end_inset + + are bearable for business cases. \end_layout \begin_layout Itemize @@ -6166,6 +6335,27 @@ We assume that the number of application instances is linearly scaling with . \end_layout +\begin_layout Itemize +We assume that the storage nodes are (almost completely) filled with data + (sectors with RAID, and/or objects with BigCluster). +\end_layout + +\begin_layout Itemize +We assume that the number of sectors / objects per storage node is +\begin_inset Quotes eld +\end_inset + +very large +\begin_inset Quotes erd +\end_inset + +. + Some examples: a logical volume of 4 TB has 1,000,000,000 sectors or object, + each 4 KB in size. + A physical storage node providing 40 TB of storage will then provide 10 + billions of sectors / objects. +\end_layout + \begin_layout Itemize For the BigCluster architecture, we assume that all objects are always distribut ed to @@ -6209,6 +6399,123 @@ In the following example, we assume a uniform object distribution to exactly '. \end_layout +\begin_layout Itemize +When random distribution / random replication methods are used at BigCluster + object stores, we assume that for any pair (or +\begin_inset Formula $k$ +\end_inset + +-tuple) of storage nodes, the total number of objects is so high that there + always +\emph on +exists +\emph default + some objects which are present at +\emph on +all +\emph default + of the nodes of any pair / +\begin_inset Formula $k$ +\end_inset + +-tuple for any reasonable (small) +\begin_inset Formula $k$ +\end_inset + +. + This means, we assume not only uniformity in random replication, but also + that the total number of objects is practically +\begin_inset Quotes eld +\end_inset + +infinite +\begin_inset Quotes erd +\end_inset + + compared to relatively small practical values of +\begin_inset Formula $k$ +\end_inset + +. +\begin_inset Newline newline +\end_inset + + +\begin_inset Graphics + filename images/MatieresCorrosives.png + lyxscale 50 + scale 17 + +\end_inset + + For mathematically interested readers: be careful when trying to argue + with the probability to hit some object intersection for some given +\begin_inset Formula $k$ +\end_inset + +-tuple of storage nodes while +\begin_inset Formula $n$ +\end_inset + + is a growing parameter. + Even when such a +\emph on +single +\emph default + probability is declining with growing both +\begin_inset Formula $k$ +\end_inset + + and +\begin_inset Formula $n$ +\end_inset + +, and even when the +\emph on +single +\emph default + probability for the existence of an intersection somewhen gets lower than + +\begin_inset Formula $1$ +\end_inset + +, this has an impact onto the +\emph on +total +\emph default + incident probability of the +\emph on +whole +\emph default + BigCluster. + In +\emph on +general +\emph default +, the +\emph on +number +\emph default + of such tuples is growing with +\begin_inset Formula $O(\binom{k\cdot n}{k})=O((k\cdot n)!)$ +\end_inset + +. + So, don't forget to sum up +\emph on +all +\emph default + probabilities even if a single one appears to be +\begin_inset Quotes eld +\end_inset + +neglectible +\begin_inset Quotes erd +\end_inset + +. +\end_layout + \begin_layout Itemize For the LocalSharding (DRBDorMARS) architecture, we assume that only local storage is used. @@ -6941,6 +7248,87 @@ undecidable Be cautious when drawing assumptions out of thin air! \end_layout +\begin_layout Standard +\noindent +\begin_inset Graphics + filename images/MatieresCorrosives.png + lyxscale 50 + scale 17 + +\end_inset + + Conversely, the assumption that +\emph on +any +\emph default + unaccessible object may halt your application, might be too strong for + +\emph on +some +\emph default + use cases. + Therefore, some practical behaviour may be inbetween the solid thin lines + and the dashed lines of some given color. + Be extremely careful when constructing such an intermediate case. + The above example of a loss rate of 1/1,000,000 of sectors in a classical + filesystem should not be extended to lower values like 1/1,000,000,000 + without knowing exactly how the filesystem works, and how it will react + +\emph on +in detail +\emph default + +\begin_inset Foot +status open + +\begin_layout Plain Layout +In general, it is insufficient to analyze the logical dependencies inside + of a filesystem instance, such as which inode contains some pointers to + which other filesystem objects, etc. + There exist further +\series bold +runtime dependencies +\series default +, such as +\family typewriter +nr_requests +\family default + block-layer restrictions on IO queue depths, and/or capabilities / limitiations + of the hardware, and so on. + Trying to model all of these influences in a reasonable way could be a + +\emph on +major +\emph default + research undertakement outside the scope of this MARS manual. +\end_layout + +\end_inset + +. + The grey zone between the extreme cases thin solid vs dashed is a +\series bold +dangerous zone +\series default +! +\end_layout + +\begin_layout Standard +\noindent +\begin_inset Graphics + filename images/lightbulb_brightlit_benj_.png + lyxscale 12 + scale 7 + +\end_inset + +If you want to stay at the +\series bold +safe side +\series default +, simply obey the fundamental law as explained in the next section: +\end_layout + \begin_layout Subsection Optimum Reliability from Architecture \begin_inset CommandInset label @@ -51784,6 +52172,5394 @@ input{screener-verbose.help} \end_layout +\begin_layout Chapter +Football Redundancy Diagrams +\begin_inset CommandInset label +LatexCommand label +name "chap:Football-Redundancy-Diagrams" + +\end_inset + + +\end_layout + +\begin_layout Standard +The following tables are showing the number of replicas during Football. + We focus at the common case of starting with 2 replicas, and ending up + in a total of another 2 replicas at another machines. + Further cases, involving multiple secondaries, should go analogously. +\end_layout + +\begin_layout Standard +Active primaries are colored in red. +\end_layout + +\begin_layout Standard +Backup or shadow replicas (which are present at LVM level, but currently + not used by MARS) are in parentheses. + In case of emergency, they could be activated again. +\end_layout + +\begin_layout Standard +Replicas which are not in parentheses are kept in +\family typewriter +UpToDate +\family default + state all the time, until they are retired into backup replicas. +\end_layout + +\begin_layout Section +Parallel +\family typewriter +migrate +\end_layout + +\begin_layout Standard +This creates two additional replicas in parallel, at the target pair. + After handover to the new site, and after some configurable waiting time, + the old replicas are deleted. +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +migrate +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate x 2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Section +Stepwise +\family typewriter +migrate +\end_layout + +\begin_layout Standard +This variant is useful for hardware lifecycle. + The uplink of the old hardware is only loaded with creation of 1 replica + in migration step 1. + Step 2 creates then another replica at the new hardware, which should have + a better replication network (e.g. + better uplinks and/or better capacity for cross-traffic between datacenters). +\end_layout + +\begin_layout Standard +This variant is selected by parameter +\family typewriter +migrate_two_phase=1 +\family default +. +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +2-Step +\begin_inset Newline newline +\end_inset + +migrate +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +3 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate 2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Section +Plain +\family typewriter +shrink +\end_layout + +\begin_layout Standard +Here we need to discriminate between replicas with the old size, and the + new size (which is typically smaller than the old size). +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +shrink +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +old_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +new_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\color inherit ++ (1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Working +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(2) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Finished +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(2) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Section +Full +\family typewriter +migrate+shrink +\end_layout + +\begin_layout Standard +This variant is +\emph on +almost +\emph default + equivalent to +\family typewriter +migrate +\family default + followed by +\family typewriter +shrink +\family default +. + The only difference is that cleanup is done +\emph on +later +\emph default +. + This means, more replicas are kept for some longer time. + Thus this variant is more safe than doing +\family typewriter +migrate +\family default + and +\family typewriter +shrink +\family default + separately. +\end_layout + +\begin_layout Standard +This variant is selected by parameters +\family typewriter +migrate_two_phase=0 +\family default + and +\family typewriter +migrate_always_all=1 +\family default + and +\family typewriter +migrate_early_cleanup=0 +\family default +: +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +FULL +\begin_inset Newline newline +\end_inset + +migrate+shrink +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +old_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +new_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate x 2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\color inherit ++ (1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Working +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(4) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Finished +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(4) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Standard +\noindent +There is a variant which does early cleanup, which is roughly equivalent + to doing a standalone +\family typewriter +migrate +\family default + followed by a standalone +\family typewriter +shrink +\family default +. + +\end_layout + +\begin_layout Standard +This variant is selected by parameters +\family typewriter +migrate_two_phase=0 +\family default + and +\family typewriter +migrate_always_all=1 +\family default + and +\family typewriter +migrate_early_cleanup=1 +\family default +. + It is less safe because it keeps less replicas and is thus less recommended: +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +Sequential +\begin_inset Newline newline +\end_inset + +migrate+shrink +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +old_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +new_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate x 2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Early Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\color inherit ++ (1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Working +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(2) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Finished +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(2) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Section +Stepwise +\family typewriter +migrate+shrink +\end_layout + +\begin_layout Standard +This variant is useful for hardware lifecycle. + The uplink of the old hardware is only loaded with creation of 1 replica + in migration step 1. + Step 2 creates then another replica at the new hardware, which should have + a better replication network. + +\end_layout + +\begin_layout Standard +This variant is selected by parameters +\family typewriter +migrate_two_phase=1 +\family default + and +\family typewriter +migrate_always_all=1 +\family default + and +\family typewriter +migrate_early_cleanup=0 +\family default +: +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +2-Step +\begin_inset Newline newline +\end_inset + +migrate+shrink +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +old_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +new_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +3 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate 2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\color inherit ++ (1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Working +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(4) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Finished +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(4) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Standard +\noindent +This variant can also be combined with early cleanup. + The result is similar to above. + The only difference is that the second additional replica is created at + the new hardware. + +\end_layout + +\begin_layout Standard +This variant is selected by parameters +\family typewriter +migrate_two_phase=1 +\family default + and +\family typewriter +migrate_always_all=0 +\family default + and +\family typewriter +migrate_early_cleanup= +\family default +1. + Again, this variant is less safe and therefore less recommended. +\end_layout + +\begin_layout Standard +However, it keeps at least 2 (backup) replicas all the time and thus could + be an alternative when decommissioning of old hardware is time-critical. +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +2-Step +\begin_inset Newline newline +\end_inset + +migrate+shrink +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +old_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +new_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +3 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate 2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +4 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Early Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\color inherit ++ (1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Working +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(2) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Finished +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(2) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Section +FAST +\family typewriter +migrate+shrink +\end_layout + +\begin_layout Standard +This variant tries to keep a balance by not creating too many unnecessary + replicas and to reduce network traffic. +\end_layout + +\begin_layout Standard +This variant is selected by parameters +\family typewriter +migrate_two_phase=0 +\family default + and +\family typewriter +migrate_always_all=0 +\family default + and +\family typewriter +migrate_early_cleanup=0 +\family default +: +\end_layout + +\begin_layout Standard +\noindent +\align center +\begin_inset Tabular + + + + + + + + + + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "14col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +FAST +\begin_inset Newline newline +\end_inset + +migrate+shrink +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +SRC +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Primary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "10col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +DST +\begin_inset Newline newline +\end_inset + +Secondary +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +old_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +\begin_inset Box Frameless +position "t" +hor_pos "c" +has_inner_box 1 +inner_pos "t" +use_parbox 0 +use_makebox 0 +width "12col%" +special "none" +height "1in" +height_special "totalheight" +thickness "0.4pt" +separation "3pt" +shadowsize "4pt" +framecolor "black" +backgroundcolor "none" +status open + +\begin_layout Plain Layout +# Replicas +\begin_inset Newline newline +\end_inset + +new_size +\end_layout + +\end_inset + + +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Migrate x 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +3 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Start +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\color inherit ++ (1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +3 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Working +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red + 1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(3) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +Shrink Finished +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(1) + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +(3) +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\begin_inset Text + +\begin_layout Plain Layout +After Cleanup +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout + +\color red +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +1 +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +- +\end_layout + +\end_inset + + +\begin_inset Text + +\begin_layout Plain Layout +2 +\end_layout + +\end_inset + + + + +\end_inset + + +\end_layout + +\begin_layout Standard +\noindent +As before, this could +\emph on +theoretically +\emph default + be combined with early cleanup. + Such a combination is however not recommended because there is one intermediate + step where all existing replicas are at the DST primary, and thus this + one machine must not fail. +\end_layout + \begin_layout Chapter GNU Free Documentation License \begin_inset CommandInset label diff --git a/docu/mars-manual.pdf b/docu/mars-manual.pdf index 2a467ace..cc7f9301 100644 Binary files a/docu/mars-manual.pdf and b/docu/mars-manual.pdf differ diff --git a/kernel/brick_wait.h b/kernel/brick_wait.h new file mode 100644 index 00000000..fb5bc1be --- /dev/null +++ b/kernel/brick_wait.h @@ -0,0 +1,69 @@ +/* + * MARS Long Distance Replication Software + * + * This file is part of MARS project: http://schoebel.github.io/mars/ + * + * Copyright (C) 2010-2014 Thomas Schoebel-Theuer + * Copyright (C) 2011-2014 1&1 Internet AG + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef BRICK_WAIT_H +#define BRICK_WAIT_H + +/* compat to some elder kernels... + */ +#ifndef ___wait_cond_timeout +#define ___wait_cond_timeout(x) (x) +#define prepare_to_wait_event(a,b,c) (prepare_to_wait(a, b, c), 0) +#endif + +/* Some code stolen from include/linux/wait.h + */ +#define brick_wait(wq, condition, timeout) \ +({ \ + __label__ __out; \ + wait_queue_t __wait; \ + long __ret = timeout; /* explicit shadow */ \ + \ + might_sleep(); \ + /* check in advance to avoid spinlocks in fastpath */ \ + if (condition) \ + goto __out; \ + \ + INIT_LIST_HEAD(&__wait.task_list); \ + __wait.flags = 0; \ + \ + for (;;) { \ + long __int = prepare_to_wait_event(&wq, &__wait, TASK_INTERRUPTIBLE); \ + \ + if (__int) { \ + __ret = __int; \ + break; \ + } \ + \ + __ret = schedule_timeout(__ret); \ + \ + __set_current_state(TASK_RUNNING); \ + if (___wait_cond_timeout(condition)) \ + break; \ + } \ + finish_wait(&wq, &__wait); \ +__out: __ret; \ +}) + + +#endif diff --git a/kernel/mars_copy.c b/kernel/mars_copy.c index 4022f663..14c2c110 100644 --- a/kernel/mars_copy.c +++ b/kernel/mars_copy.c @@ -245,27 +245,19 @@ void copy_endio(struct generic_callback *cb) error = -EINVAL; goto exit; } - st->active[queue] = false; - if (unlikely(st->table[queue])) { + if (unlikely(st->table[queue] != mref)) { MARS_ERR("table corruption at %d %d (%p => %p)\n", index, queue, st->table[queue], mref); error = -EEXIST; goto exit; } if (unlikely(cb->cb_error < 0)) { error = cb->cb_error; - __clear_mref(brick, mref, queue); /* This is racy, but does no harm. * Worst case just produces more error output. */ if (!brick->copy_error_count++) { MARS_WRN("IO error %d on index %d, old state = %d\n", cb->cb_error, index, st->state); } - } else { - if (unlikely(st->table[queue])) { - MARS_ERR("overwriting index %d, state = %d\n", index, st->state); - _clear_mref(brick, index, queue); - } - st->table[queue] = mref; } exit: @@ -273,6 +265,7 @@ exit: st->error = error; _clash(brick); } + st->active[queue] = false; if (mref->ref_rw) { atomic_dec(&brick->copy_write_flight); atomic_dec(&global_copy_write_flight); @@ -294,6 +287,7 @@ int _make_mref(struct copy_brick *brick, int index, int queue, void *data, loff_ struct mref_object *mref; struct copy_mref_aspect *mref_a; struct copy_input *input; + struct copy_state *st; int offset; int len; int status = -EAGAIN; @@ -353,7 +347,10 @@ int _make_mref(struct copy_brick *brick, int index, int queue, void *data, loff_ //MARS_IO("queue = %d index = %d pos = %lld len = %d rw = %d\n", queue, index, mref->ref_pos, mref->ref_len, rw); - GET_STATE(brick, index).active[queue] = true; + st = &GET_STATE(brick, index); + st->table[queue] = mref; + st->active[queue] = true; + if (rw) { atomic_inc(&brick->copy_write_flight); atomic_inc(&global_copy_write_flight); @@ -442,8 +439,6 @@ restart: goto idle; } - _clear_mref(brick, index, 1); - _clear_mref(brick, index, 0); st->writeout = false; st->error = 0; @@ -475,17 +470,16 @@ restart: next_state = COPY_STATE_READ2; /* fallthrough */ case COPY_STATE_READ2: - mref1 = st->table[1]; - if (!mref1) { // idempotence: wait by unchanged state + if (st->active[1]) { // idempotence: wait by unchanged state goto idle; } /* fallthrough => wait for both mrefs to appear */ case COPY_STATE_READ1: case COPY_STATE_READ3: - mref0 = st->table[0]; - if (!mref0) { // idempotence: wait by unchanged state + if (st->active[0]) { // idempotence: wait by unchanged state goto idle; } + mref0 = st->table[0]; if (brick->copy_limiter) { int amount = (mref0->ref_len - 1) / 1024 + 1; mars_limit_sleep(brick->copy_limiter, amount); @@ -565,6 +559,11 @@ restart: progress = -EILSEQ; break; } + if (unlikely(st->active[0])) { + MARS_ERR("src buffer for write is active, state %d at index %d\n", state, index); + progress = -EILSEQ; + break; + } if (unlikely(brick->is_aborting)) { progress = -EINTR; break; @@ -587,8 +586,7 @@ restart: next_state = COPY_STATE_WRITTEN; /* fallthrough */ case COPY_STATE_WRITTEN: - mref1 = st->table[1]; - if (!mref1) { // idempotence: wait by unchanged state + if (st->active[1]) { // idempotence: wait by unchanged state MARS_IO("irrelevant\n"); goto idle; } diff --git a/kernel/mars_trans_logger.c b/kernel/mars_trans_logger.c index 9e20a19b..7de517cd 100644 --- a/kernel/mars_trans_logger.c +++ b/kernel/mars_trans_logger.c @@ -36,6 +36,7 @@ #include #include +#include "brick_wait.h" #include "mars.h" #include "lib_limiter.h" @@ -735,7 +736,7 @@ int _write_ref_get(struct trans_logger_output *output, struct trans_logger_mref_ #ifdef DELAY_CALLERS // delay in case of too many master shadows / memory shortage - wait_event_interruptible_timeout(brick->caller_event, + brick_wait(brick->caller_event, !brick->delay_callers && (brick_global_memlimit < 1024 || atomic64_read(&global_mshadow_used) / 1024 < brick_global_memlimit), HZ / 2); @@ -2300,7 +2301,7 @@ int _do_ranking(struct trans_logger_brick *brick) } } else if (brick->delay_callers) { brick->delay_callers = false; - wake_up_interruptible(&brick->caller_event); + wake_up_interruptible_all(&brick->caller_event); } // global limit for flying mrefs @@ -2605,7 +2606,7 @@ void trans_logger_log(struct trans_logger_brick *brick) int winner; int nr; - wait_event_interruptible_timeout( + brick_wait( brick->worker_event, ({ winner = _do_ranking(brick); @@ -2771,7 +2772,7 @@ void wait_replay(struct trans_logger_brick *brick, struct trans_logger_mref_aspe bool ok = false; bool was_empty; - wait_event_interruptible_timeout(brick->worker_event, + brick_wait(brick->worker_event, atomic_read(&brick->replay_count) < max && (_has_conflict(brick, mref_a) ? conflicts++ : (ok = true), ok), 60 * HZ); diff --git a/userspace/marsadm b/userspace/marsadm index 22d046d2..b78381d4 100755 --- a/userspace/marsadm +++ b/userspace/marsadm @@ -1697,7 +1697,6 @@ sub try_to_avoid_splitbrain { my $old_timeout = $timeout; $timeout = $window if $timeout < 0; $old_primary = "" if $old_primary eq "(none)"; - wait_cluster($cmd, $res, $old_primary); if (!detect_splitbrain($res, 0)) { lwarn "ATTENTION: you are starting a non-forced primary switchover in a split brain situation.\n"; lwarn "ATTENTION: that's no good idea.\n";