[pve-devel] [PATCH ha-manager v3 3/6] add Fence class for external fence devices

[Thomas Lamprecht]

This class provides methods for starting and checking the current
status of a fence job.

When a fence job is started we execute a fence agent command.
If we can fork this happens in forked worker, which can be multiple
processes also, when parallel devices are configured.

When a device fails to successfully fence a node we try the next
device configured, or if no device is left we tell the CRM and let
him decide what to do.

If one process of a parallel device set fails we kill the remaining
(with reset_hard) and try the next device as we want to avoid a
partial fenced node. (not ideal, user should see status of devices)

The forked processes need to be
The current running fence jobs can be picked up (if env. allows
forking) and processed by calling the process_fencing method.

If the CRM (which should handle the fencing) looses its lock
bail_out can be called to kill all fencing processes and reset
the fencing status of all nodes.

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/PVE/HA/Fence.pm | 217 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/PVE/HA/Makefile |   2 +-
 2 files changed, 218 insertions(+), 1 deletion(-)
 create mode 100644 src/PVE/HA/Fence.pm

diff --git a/src/PVE/HA/Fence.pm b/src/PVE/HA/Fence.pm
new file mode 100644
index 0000000..f4eff10
--- /dev/null
+++ b/src/PVE/HA/Fence.pm
@@ -0,0 +1,217 @@
+package PVE::HA::Fence;
+
+use strict;
+use warnings;
+use POSIX qw( WNOHANG );
+use PVE::HA::FenceConfig;
+use Data::Dumper;
+
+
+ # pid's and additional info of fence processes
+my $fence_jobs = {};
+
+# fence state of a node
+my $fenced_nodes = {};
+
+sub has_fencing_job { # update for parallel fencing
+    my ($node) = @_;
+
+    foreach my $job (values %$fence_jobs) {
+	return 1 if ($job->{node} eq $node);
+    }
+    return undef;
+}
+
+my $virtual_pid = 0; # hack for test framework
+
+sub start_fencing {
+    my ($haenv, $node, $try) = @_;
+
+    $try = 0 if !defined($try) || $try<0;
+
+    my $fence_cfg = $haenv->read_fence_config();
+    my $commands = PVE::HA::FenceConfig::get_commands($node, $try, $fence_cfg);
+
+    if (!$commands) {
+	$haenv->log('err', "no commands for node '$node'");
+	$fenced_nodes->{$node}->{failure} = 1;
+	return 0;
+    }
+
+    my $can_fork = ($haenv->get_max_workers() > 0) ? 1 : 0;
+    my $hw = $haenv->hardware();
+
+    $fenced_nodes->{$node}->{needed} = scalar @$commands;
+    $fenced_nodes->{$node}->{triggered} = 0;
+
+    for my $cmd (@$commands)
+    {
+	my $cmd_str = "$cmd->{agent} " .
+	    PVE::HA::FenceConfig::gen_arg_str(@{$cmd->{param}});
+
+	if ($can_fork) {
+	    my $pid = fork();
+	    if (!defined($pid)) {
+		$haenv->log('err', "forking fence job failed");
+		return 0;
+	    } elsif ($pid==0) { # child
+		$hw->exec_fence_agent($cmd->{agent}, $node, @{$cmd->{param}});
+		exit(-1);
+	    } else {
+		$fence_jobs->{$pid} = {cmd=>$cmd_str, node=>$node, try=>$try};
+	    }
+	} else {
+	    my $res = -1;
+	    eval {
+		$res = $hw->exec_fence_agent($cmd->{agent}, $node, @{$cmd->{param}});
+		$res = $res << 8 if $res > 0;
+	    };
+	    if (my $err = $@) {
+		$haenv->log('err', $err);
+	    }
+
+	    $virtual_pid++;
+	    $fence_jobs->{$virtual_pid} = {cmd => $cmd_str, node => $node,
+					   try => $try, ec => $res};
+	}
+    }
+
+    return 1;
+}
+
+
+# check childs and process exit status
+my $check_jobs = sub {
+    my ($haenv) = @_;
+
+    my $succeeded = {};
+    my $failed = {};
+
+    my @finished = ();
+
+    # pick up all finsihed childs if we can fork
+    if ($haenv->get_max_workers() > 0) {
+	while((my $res = waitpid(-1, WNOHANG))>0) {
+	    $fence_jobs->{$res}->{ec} = $? if $fence_jobs->{$res};
+	    push @finished, $res;
+	}
+    } else {
+	@finished = keys %{$fence_jobs};
+    }
+
+    #    while((my $res = waitpid(-1, WNOHANG))>0) {
+    foreach my $res (@finished) {
+	if (my $job = $fence_jobs->{$res}) {
+	    my $ec = $job->{ec};
+
+	    my $status = {
+		exit_code => $ec,
+		cmd => $job->{cmd},
+		try => $job->{try}
+	    };
+
+	    if ($ec == 0) {
+		$succeeded->{$job->{node}} = $status;
+	    } else {
+		$failed->{$job->{node}} = $status;
+	    }
+
+	    delete $fence_jobs->{$res};
+
+	} else {
+	    warn "exit from unknown child (PID=$res)";
+	}
+
+    }
+
+    return ($succeeded, $failed);
+};
+
+
+my $reset_hard = sub {
+    my ($haenv, $node) = @_;
+
+    while (my ($pid, $job) = each %$fence_jobs) {
+	next if $job->{node} ne $node;
+
+	if ($haenv->max_workers() > 0) {
+	    kill KILL => $pid;
+	    # fixme maybe use an timeout even if kill should not hang?
+	    waitpid($pid, 0); # pick it up directly
+	}
+	delete $fence_jobs->{$pid};
+    }
+
+    delete $fenced_nodes->{$node} if $fenced_nodes->{$node};
+};
+
+
+# pick up jobs and process them
+sub process_fencing {
+    my ($haenv) = @_;
+
+    my $fence_cfg = $haenv->read_fence_config();
+
+    my ($succeeded, $failed) = &$check_jobs($haenv);
+
+    foreach my $node (keys %$succeeded) {
+	# count how many fence devices succeeded
+	# this is needed for parallel devices
+	$fenced_nodes->{$node}->{triggered}++;
+    }
+
+    # try next device for failed jobs
+    while(my ($node, $job) = each %$failed) {
+	$haenv->log('err', "fence job failed: '$job->{cmd}' returned '$job->{exit_code}'");
+
+	while($job->{try} < PVE::HA::FenceConfig::count_devices($node, $fence_cfg) )
+	{
+	    &$reset_hard($haenv, $node);
+	    $job->{try}++;
+
+	    return if start_fencing($node, $job->{try});
+
+	    $haenv->log('warn', "Couldn't start fence try '$job->{try}'");
+	}
+
+	    $haenv->log('err', "Tried all fence devices\n");
+	    # fixme: returnproper exit code so CRM waits for the agent lock
+    }
+}
+
+
+sub is_node_fenced {
+    my ($node) = @_;
+
+    my $state = $fenced_nodes->{$node};
+    return 0 if !$state;
+
+    return -1 if $state->{failure} && $state->{failure} == 1;
+
+    return ($state->{needed} && $state->{triggered} &&
+	   $state->{triggered} >= $state->{needed}) ? 1 : 0;
+}
+
+
+sub reset {
+    my ($node, $noerr) = @_;
+
+    delete $fenced_nodes->{$node} if $fenced_nodes->{$node};
+}
+
+
+sub bail_out {
+    my ($haenv) = @_;
+
+    if ($haenv->max_workers() > 0) {
+	foreach my $pid (keys %$fence_jobs) {
+	    kill KILL => $pid;
+	    waitpid($pid, 0); # has to come back directly
+	}
+    }
+
+    $fenced_nodes = {};
+    $fence_jobs = {};
+}
+
+1;
diff --git a/src/PVE/HA/Makefile b/src/PVE/HA/Makefile
index 839300c..436198f 100644
--- a/src/PVE/HA/Makefile
+++ b/src/PVE/HA/Makefile
@@ -1,5 +1,5 @@
 SOURCES=CRM.pm Env.pm Groups.pm Resources.pm Config.pm LRM.pm Manager.pm \
-	NodeStatus.pm Tools.pm FenceConfig.pm
+	NodeStatus.pm Tools.pm FenceConfig.pm Fence.pm
 
 .PHONY: install
 install:
-- 
2.1.4