From cda71f2746d623d495ffc4e52778e895d77942e2 Mon Sep 17 00:00:00 2001 From: Thomas Schoebel-Theuer Date: Thu, 9 Jul 2020 14:46:23 +0200 Subject: [PATCH] marsadm: new command primary+prosumer --- userspace/marsadm | 171 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 143 insertions(+), 28 deletions(-) diff --git a/userspace/marsadm b/userspace/marsadm index 3573ee10..8e87333e 100755 --- a/userspace/marsadm +++ b/userspace/marsadm @@ -2777,10 +2777,12 @@ sub wait_cluster { } sub wait_cluster_noforce { - my ($cmd, $res) = @_; + my ($cmd, $res, $hosts) = @_; if (!$force) { - lprint "WAITING for communication\n" if $verbose; - wait_cluster($cmd, $res, "*", 0); + $res = "all" if (!$res || $res =~ m/,/); + $hosts = "*" unless $hosts; + lprint "WAITING for communication with '$hosts'\n" if $verbose; + wait_cluster($cmd, $res, $hosts, 0); } } @@ -4040,6 +4042,16 @@ sub _get_designated_primary { return $val; } +my %pri_new; + +sub _get_future_primary { + my ($cmd, $res, $unchecked) = @_; + if ($pri_new{$res}) { + return $pri_new{$res}; + } + return _get_designated_primary(@_); +} + sub is_actual_primary { my ($cmd, $res, $peer) = @_; $peer = $host unless (defined($peer) && $peer); @@ -6431,6 +6443,9 @@ sub primary_phase0 { } } lprint "Current designated primary: $old\n"; + lprint "Future designated primary: $new\n"; + $pri_old{$res} = $old; + $pri_new{$res} = $new; if ($cmd =~ m/primary/) { if ($host ne $old) { lprint "Allowing handover in cases of sync: ignore_sync=$ignore_sync\n" if $ignore_sync; @@ -6518,7 +6533,7 @@ sub primary_phase0 { # otherwise: prepare prosumers sub primary_phase0a { my ($cmd, $res) = @_; - my $new = $host; + my $new = $pri_new{$res}; if (!$force && $cmd =~ m/primary/) { lprint "Prepare new primary '$new' handover\n"; _switch($cmd, $res, "$mars/resource-$res/todo-$new/fetch", 1); @@ -6715,8 +6730,8 @@ sub primary_phase1b { return 0 if $force; my $check_logger = 0; my $this_stamp = 0; - my $old = _get_designated_primary($cmd, $res, -1); - my $new = $host; + my $old = $pri_old{$res}; + my $new = $pri_new{$res}; if (todo_local(@_)) { my $status = check_primary_gone($cmd, $res, $old); return $status if $status; @@ -6824,9 +6839,8 @@ sub primary_phase2b { sub primary_phase3 { my ($cmd, $res) = @_; return 0 unless $cmd =~ m/primary/; - my $old = _get_designated_primary($cmd, $res, -1); - $pri_old{$res} = $old; - my $new = $host; + my $old = $pri_old{$res}; + my $new = $pri_new{$res}; _primary_res($res, $new, $old); $allow_fail_action = \&compensate_primary_fail_switched; my $prosumers = get_prosumers(@_); @@ -6902,6 +6916,7 @@ sub primary_phase3c { $allow_fail_action = \&compensate_primary_fail_prepared; _trigger(3); } + return 0; } # wait for prosumer-handover finished @@ -6909,7 +6924,7 @@ sub primary_phase3d { my ($cmd, $res) = @_; return 0 if $force || todo_local(@_); my $old = $pri_old{$res}; - my $new = $host; + my $new = $pri_new{$res}; my $prosumers = get_prosumers(@_); foreach my $peer (split("\\+", $prosumers)) { my $lnk = "$mars/resource-$res/actual-$peer/prosumer-on"; @@ -6985,7 +7000,7 @@ sub primary_phase3f { my ($cmd, $res) = @_; if (!$force && !todo_local(@_)) { my $old = $pri_old{$res}; - my $new = $host; + my $new = $pri_new{$res}; if ($old ne $new) { my $lnk = "$mars/resource-$res/actual-$old/is-primary"; my $val = get_link($lnk, 1); @@ -7032,12 +7047,14 @@ sub primary_phase4 { check_mars_device($cmd, $res, 1, 0); } elsif (!$force) { my $old = $pri_old{$res}; - my $new = $host; - lprint "Unexporting old primary '$old' for safety.\n"; - my $lnk = "$mars/resource-$res/todo-$old/exports"; - set_link("(none)", $lnk); - $lnk = "$mars/resource-$res/todo-$old/multi-prosumer"; - set_link("0", $lnk); + my $new = $pri_new{$res}; + if ($old && $new && $new ne $old && $old ne "(none)") { + lprint "Unexporting old primary '$old' for safety.\n"; + my $lnk = "$mars/resource-$res/todo-$old/exports"; + set_link("(none)", $lnk); + $lnk = "$mars/resource-$res/todo-$old/multi-prosumer"; + set_link("0", $lnk); + } } # new switch semantics, when nothing has failed before: up up_res_phase1(@_); @@ -7078,19 +7095,34 @@ my %pros_umount; sub prosumer_phase0 { my ($cmd, $res) = @_; - my $primary = _get_designated_primary($cmd, $res, 0); - lprint "Current designated primary: $primary\n"; - if ($primary eq "(none)") { + my $max_retry = 3; + retry: + my $old_primary = _get_designated_primary($cmd, $res, 0); + my $new_primary = _get_future_primary($cmd, $res, 0); + lprint "Current designated primary: $old_primary\n"; + lprint "Future designated primary: $new_primary\n"; + if ($new_primary eq "(none)") { my $old_prosumers = _get_prosumer(@_); my $new_prosumers = parse_list_spec($cmd_suffix{"prosumer"}, $old_prosumers, $host); if ($new_prosumers ne "(local)" && $new_prosumers ne "(none)") { - ldie "Cannot activate prosumers '$new_prosumers' when there is no designated primary.\n"; + if ($max_retry-- > 0) { + lwarn "cannot find a designated primary.\n"; + update_cluster($cmd, $res); + goto retry; + } + ldie "Cannot activate prosumers '$new_prosumers' without a designated primary.\n"; } } - if ($primary ne "(none)" && - !is_actual_primary($cmd, $res, $primary)) { - ldie "Designated primary '$primary' is not actually primary\n" unless $force; - lwarn "CONTINUE AT YOUR RISK, although designated primary '$primary' is not actually primary\n"; + if ($old_primary ne "(none)" && + !is_actual_primary($cmd, $res, $old_primary)) { + if ($max_retry-- > 0) { + lwarn "current designated primary '$old_primary' is not actually primary\n"; + sleep(1); + wait_cluster_noforce($cmd, $res, $old_primary); + goto retry; + } + ldie "Designated primary '$old_primary' is not actually primary\n" unless $force; + lwarn "CONTINUE AT YOUR RISK, although current designated primary '$old_primary' is not actually primary\n"; } my $old_prosumers = _get_prosumer(@_); @@ -7109,8 +7141,8 @@ sub prosumer_phase0 { $old_prosumers = ""; $old_means .= " means ''"; } elsif ($old_prosumers eq "(local)") { - $old_prosumers = $primary; - $old_means .= " means designated primary '$primary'"; + $old_prosumers = $new_primary; + $old_means .= " means designated primary '$new_primary'"; } if ($new_prosumers eq "(none)") { $new_prosumers = ""; @@ -7205,7 +7237,7 @@ sub prosumer_phase0 { sub prosumer_phase1 { my ($cmd, $res) = @_; - my $primary = _get_designated_primary($cmd, $res, 0); + my $primary = _get_future_primary($cmd, $res, 0); my $new_prosumers = $pros_new{$res}; my $uni_prosumers = $pros_uni{$res}; my $inter_prosumers = $pros_inter{$res}; @@ -10815,6 +10847,89 @@ my %cmd_table = \&prosumer_phase5, "set final prosumers", + "LOOP", + \&prosumer_phase6, + "wait for device", + ], + "primary+prosumer" + => [ + "usage: primary= prosumer= []", + " where is '+'-separated.", + " Operators += and -= are also allowed in place of = .", + "Switch both the primary and the prosumer device at the same time.", + [ + \&update_guests, + ], + + \&primary_phase0, + "check primary preconditions", + \&prosumer_phase0, + "check prosumer preconditions", + + "FORK", + "LOOP", + \&prosumer_phase1, + "remove old prosumers", + + \&primary_phase0a, + "conditionally wait for fetch off", + + "LOOP", + \&primary_phase0b, + "wait for systemd", + + "LOOP", + \&prosumer_phase2, + "check that any old prosumers are gone when necessary", + + "LOOP", + \&primary_phase1, + "leave primary state", + + "LOOP", + \&primary_phase1b, + "trigger remote", + + "LOOP", + \&primary_phase2, + "wait for cluster when necessary", + + "LOOP", + \&primary_phase2b, + "avoid split brain", + + \&primary_phase3, + "switch to primary", + + "LOOP", + \&primary_phase3b, + "trigger remote", + + \&primary_phase3c, + "trigger prosumer handover", + + \&prosumer_phase3, + "set new prosumers", + + "LOOP", + \&primary_phase3d, + "wait for prosumer handover and open gate", + + "LOOP", + \&primary_phase3e, + "wait for gate open and reset old primary exports", + + "LOOP", + \&primary_phase3f, + "wait for old primary gone", + + "LOOP", + \&prosumer_phase4, + "check that all old prosumers are finally gone", + + \&prosumer_phase5, + "set final prosumers", + "LOOP", \&prosumer_phase6, "wait for device",