[pve-devel] [RFC pve-ha-manager 2/5] add Fence class for external fence devices

[Thomas Lamprecht]

This class provides methods for starting and checking the current
status of a fence job.

When a fence job is started we fork a process which execs the fence
agent command. This can be multiple processes also, when parallel
devices are configured.
When a device fails to successfully fence a node we try the next device
configured, or if no device is left we fall back to the current
procedure and wait until we got the lrm agent lock of the node.

If one process of a parallel device set fails we kill the remaining
(with reset_hard) and try the next device. (not ideal, needs to be
logged detailed)

The forked processes need to be picked up and processed by calling
the process_fencing method.

If the CRM (which should handle the fencing) looses its lock
bail_out must be called to kill all fencing processes and reset
the fencing status.

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/PVE/HA/Fence.pm | 179 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/PVE/HA/Makefile |   2 +-
 2 files changed, 180 insertions(+), 1 deletion(-)
 create mode 100644 src/PVE/HA/Fence.pm

diff --git a/src/PVE/HA/Fence.pm b/src/PVE/HA/Fence.pm
new file mode 100644
index 0000000..b9b5eac
--- /dev/null
+++ b/src/PVE/HA/Fence.pm
@@ -0,0 +1,179 @@
+package PVE::HA::Fence;
+
+use strict;
+use warnings;
+use POSIX qw( WNOHANG );
+use PVE::HA::FenceConfig;
+use Data::Dumper;
+
+
+ # pid's and additional info of fence processes
+my $fence_jobs = {};
+
+# fence state of a node
+my $fenced_nodes = {};
+
+sub has_fencing_job { # update for parallel fencing
+    my ($node) = @_;
+
+    foreach my $job (values %$fence_jobs) {
+	return 1 if ($job->{node} eq $node);
+    }
+    return undef;
+}
+
+
+sub start_fencing {
+    my ($node, $try) = @_;
+
+#    warn "Trying to fence '$node' with active fence jobs" if has_fencing_job($node);
+
+    $try = 0 if !defined($try) || $try<0;
+
+    my @commands = PVE::HA::FenceConfig::get_commands($node, $try);
+
+    if (!@commands) {
+	warn "no commands for node '$node'";
+	$fenced_nodes->{$node}->{failure} = 1;
+	return 0;
+    }
+
+    $fenced_nodes->{$node}->{needed} = scalar @commands;
+    $fenced_nodes->{$node}->{triggered} = 0;
+
+    for my $cmd (@commands)
+    {
+	my $pid = fork();
+	if (!defined($pid)) {
+	    die "forking fence job failed\n";
+	} elsif ($pid==0) { # child
+	    # forward fence agent streams to our logfile
+	    $cmd .= " | sed 's/^/[test-agent]: /' ".
+		">>/var/log/pve-fence-".time.".log 2>&1";
+	    exec($cmd);
+	    exit(-1);
+	} else {
+	    $fence_jobs->{$pid} = {cmd=>$cmd, node=>$node, try=>$try};
+	}
+    }
+
+    return 1;
+}
+
+
+# check childs and process exit status
+my $check_jobs = sub {
+    my $succeeded = {};
+    my $failed = {};
+
+    while((my $res = waitpid(-1, WNOHANG))>0) {
+	my $exit_code = $?;
+
+	if (my $job = $fence_jobs->{$res}) {
+
+	    my $status = {
+		exit_code => $exit_code,
+		cmd => $job->{cmd},
+		try => $job->{try}
+	    };
+
+	    if ($exit_code==0) {
+		$succeeded->{$job->{node}} = $status;
+	    } else {
+		$failed->{$job->{node}} = $status;
+	    }
+
+	    delete $fence_jobs->{$res};
+
+	} else {
+	    warn "exit from unknown child with PID '$res'";
+	}
+
+    }
+
+    return ($succeeded, $failed);
+};
+
+
+my $reset_hard = sub {
+    my ($node) = @_;
+
+    while (my ($pid, $job) = each %$fence_jobs) {
+	next if $job->{node} ne $node;
+
+	kill KILL => $pid;
+	delete $fence_jobs->{$pid};
+	# fixme maybe use an timeout even if kill should not hang?
+	waitpid($pid, 0); # pick it up directly
+    }
+
+    delete $fenced_nodes->{$node} if $fenced_nodes->{$node};
+};
+
+
+# pick up jobs and process them
+sub proccess_fencing {
+    my ($succeeded, $failed) = &$check_jobs();
+
+    # fixme extensively log fence process outcomes
+
+    foreach my $node (keys %$succeeded) {
+	# count how many fence devices succeeded
+	# this is needed for parallel devices
+	$fenced_nodes->{$node}->{triggered}++;
+    }
+
+     # try next device for failed jobs
+    while(my ($node, $job) = each %$failed) {
+	warn "Fence job failed: '$job->{cmd}' returned '$job->{exit_code}'\n";
+
+	while($job->{try} < PVE::HA::FenceConfig::count_devices($node) )
+	{
+	    &$reset_hard($node);
+	    $job->{try}++;
+
+	    return if start_fencing($node, $job->{try});
+
+	    warn "Couldn't start fence try '$job->{try}'\n";
+	}
+
+	    warn "Tried all fence devices\n";
+	    # fixme: returnproper exit code so CRM waits for the agent lock
+    }
+}
+
+
+sub is_node_fenced {
+    my ($node) = @_;
+
+    my $state = $fenced_nodes->{$node};
+    return 0 if !$state;
+
+    return -1 if $state->{failure} && $state->{failure} == 1;
+
+    return $state->{needed} && $state->{triggered} &&
+	   $state->{triggered} >= $state->{needed};
+}
+
+
+sub reset {
+    my ($node, $noerr) = @_;
+
+    die "node has fence jobs running" if has_fencing_job($node) && !$noerr;
+
+    delete $fenced_nodes->{$node} if $fenced_nodes->{$node};
+}
+
+
+sub bail_out {
+
+    foreach my $pid (keys %$fence_jobs) {
+	kill KILL => $pid;
+	waitpid($pid, 0); # has to come back directly
+    }
+
+    $fenced_nodes = {};
+    $fence_jobs = {};
+}
+
+1;
diff --git a/src/PVE/HA/Makefile b/src/PVE/HA/Makefile
index b9902f6..5bef825 100644
--- a/src/PVE/HA/Makefile
+++ b/src/PVE/HA/Makefile
@@ -1,4 +1,4 @@
-SOURCES=CRM.pm Env.pm Groups.pm Resources.pm Config.pm LRM.pm Manager.pm NodeStatus.pm Tools.pm FenceConfig.pm
+SOURCES=CRM.pm Env.pm Groups.pm Resources.pm Config.pm LRM.pm Manager.pm NodeStatus.pm Tools.pm Fence.pm FenceConfig.pm
 
 .PHONY: install
 install:
-- 
2.1.4