[pve-devel] [RFC ha-manager v2 4/7] add Fence class for external fence devices

[Thomas Lamprecht]

This class provides methods for starting and checking the current
status of a fence job.

When a fence job is started we execute a fence agent command.
If we can fork this happens in forked worker, which can be multiple
processes also, when parallel devices are configured.

When a device fails to successfully fence a node we try the next
device configured, or if no device is left we tell the CRM and let
him decide what to do.

If one process of a parallel device set fails we kill the remaining
(with reset_hard) and try the next device as we want to avoid a
partial fenced node. (not ideal, user should see status of devices)

The current running fence jobs can be picked up (if env. allows
forking) and processed by calling the process_fencing method.

If the CRM (which should handle the fencing) looses its lock
bail_out can be called to kill all fencing processes and reset
the fencing status of all nodes.
---
 src/PVE/HA/Fence.pm | 217 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/PVE/HA/Makefile |   2 +-
 2 files changed, 218 insertions(+), 1 deletion(-)
 create mode 100644 src/PVE/HA/Fence.pm

diff --git a/src/PVE/HA/Fence.pm b/src/PVE/HA/Fence.pm
new file mode 100644
index 0000000..f4eff10
--- /dev/null
+++ b/src/PVE/HA/Fence.pm
@@ -0,0 +1,217 @@
+package PVE::HA::Fence;
+
+use strict;
+use warnings;
+use POSIX qw( WNOHANG );
+use PVE::HA::FenceConfig;
+use Data::Dumper;
+
+
+ # pid's and additional info of fence processes
+my $fence_jobs = {};
+
+# fence state of a node
+my $fenced_nodes = {};
+
+sub has_fencing_job { # update for parallel fencing
+    my ($node) = @_;
+
+    foreach my $job (values %$fence_jobs) {
+	return 1 if ($job->{node} eq $node);
+    }
+    return undef;
+}
+
+my $virtual_pid = 0; # hack for test framework
+
+sub start_fencing {
+    my ($haenv, $node, $try) = @_;
+
+    $try = 0 if !defined($try) || $try<0;
+
+    my $fence_cfg = $haenv->read_fence_config();
+    my $commands = PVE::HA::FenceConfig::get_commands($node, $try, $fence_cfg);
+
+    if (!$commands) {
+	$haenv->log('err', "no commands for node '$node'");
+	$fenced_nodes->{$node}->{failure} = 1;
+	return 0;
+    }
+
+    my $can_fork = ($haenv->get_max_workers() > 0) ? 1 : 0;
+    my $hw = $haenv->hardware();
+
+    $fenced_nodes->{$node}->{needed} = scalar @$commands;
+    $fenced_nodes->{$node}->{triggered} = 0;
+
+    for my $cmd (@$commands)
+    {
+	my $cmd_str = "$cmd->{agent} " .
+	    PVE::HA::FenceConfig::gen_arg_str(@{$cmd->{param}});
+
+	if ($can_fork) {
+	    my $pid = fork();
+	    if (!defined($pid)) {
+		$haenv->log('err', "forking fence job failed");
+		return 0;
+	    } elsif ($pid==0) { # child
+		$hw->exec_fence_agent($cmd->{agent}, $node, @{$cmd->{param}});
+		exit(-1);
+	    } else {
+		$fence_jobs->{$pid} = {cmd=>$cmd_str, node=>$node, try=>$try};
+	    }
+	} else {
+	    my $res = -1;
+	    eval {
+		$res = $hw->exec_fence_agent($cmd->{agent}, $node, @{$cmd->{param}});
+		$res = $res << 8 if $res > 0;
+	    };
+	    if (my $err = $@) {
+		$haenv->log('err', $err);
+	    }
+
+	    $virtual_pid++;
+	    $fence_jobs->{$virtual_pid} = {cmd => $cmd_str, node => $node,
+					   try => $try, ec => $res};
+	}
+    }
+
+    return 1;
+}
+
+
+# check childs and process exit status
+my $check_jobs = sub {
+    my ($haenv) = @_;
+
+    my $succeeded = {};
+    my $failed = {};
+
+    my @finished = ();
+
+    # pick up all finsihed childs if we can fork
+    if ($haenv->get_max_workers() > 0) {
+	while((my $res = waitpid(-1, WNOHANG))>0) {
+	    $fence_jobs->{$res}->{ec} = $? if $fence_jobs->{$res};
+	    push @finished, $res;
+	}
+    } else {
+	@finished = keys %{$fence_jobs};
+    }
+
+    #    while((my $res = waitpid(-1, WNOHANG))>0) {
+    foreach my $res (@finished) {
+	if (my $job = $fence_jobs->{$res}) {
+	    my $ec = $job->{ec};
+
+	    my $status = {
+		exit_code => $ec,
+		cmd => $job->{cmd},
+		try => $job->{try}
+	    };
+
+	    if ($ec == 0) {
+		$succeeded->{$job->{node}} = $status;
+	    } else {
+		$failed->{$job->{node}} = $status;
+	    }
+
+	    delete $fence_jobs->{$res};
+
+	} else {
+	    warn "exit from unknown child (PID=$res)";
+	}
+
+    }
+
+    return ($succeeded, $failed);
+};
+
+
+my $reset_hard = sub {
+    my ($haenv, $node) = @_;
+
+    while (my ($pid, $job) = each %$fence_jobs) {
+	next if $job->{node} ne $node;
+
+	if ($haenv->max_workers() > 0) {
+	    kill KILL => $pid;
+	    # fixme maybe use an timeout even if kill should not hang?
+	    waitpid($pid, 0); # pick it up directly
+	}
+	delete $fence_jobs->{$pid};
+    }
+
+    delete $fenced_nodes->{$node} if $fenced_nodes->{$node};
+};
+
+
+# pick up jobs and process them
+sub process_fencing {
+    my ($haenv) = @_;
+
+    my $fence_cfg = $haenv->read_fence_config();
+
+    my ($succeeded, $failed) = &$check_jobs($haenv);
+
+    foreach my $node (keys %$succeeded) {
+	# count how many fence devices succeeded
+	# this is needed for parallel devices
+	$fenced_nodes->{$node}->{triggered}++;
+    }
+
+    # try next device for failed jobs
+    while(my ($node, $job) = each %$failed) {
+	$haenv->log('err', "fence job failed: '$job->{cmd}' returned '$job->{exit_code}'");
+
+	while($job->{try} < PVE::HA::FenceConfig::count_devices($node, $fence_cfg) )
+	{
+	    &$reset_hard($haenv, $node);
+	    $job->{try}++;
+
+	    return if start_fencing($node, $job->{try});
+
+	    $haenv->log('warn', "Couldn't start fence try '$job->{try}'");
+	}
+
+	    $haenv->log('err', "Tried all fence devices\n");
+	    # fixme: returnproper exit code so CRM waits for the agent lock
+    }
+}
+
+
+sub is_node_fenced {
+    my ($node) = @_;
+
+    my $state = $fenced_nodes->{$node};
+    return 0 if !$state;
+
+    return -1 if $state->{failure} && $state->{failure} == 1;
+
+    return ($state->{needed} && $state->{triggered} &&
+	   $state->{triggered} >= $state->{needed}) ? 1 : 0;
+}
+
+
+sub reset {
+    my ($node, $noerr) = @_;
+
+    delete $fenced_nodes->{$node} if $fenced_nodes->{$node};
+}
+
+
+sub bail_out {
+    my ($haenv) = @_;
+
+    if ($haenv->max_workers() > 0) {
+	foreach my $pid (keys %$fence_jobs) {
+	    kill KILL => $pid;
+	    waitpid($pid, 0); # has to come back directly
+	}
+    }
+
+    $fenced_nodes = {};
+    $fence_jobs = {};
+}
+
+1;
diff --git a/src/PVE/HA/Makefile b/src/PVE/HA/Makefile
index d5f655f..d470dd9 100644
--- a/src/PVE/HA/Makefile
+++ b/src/PVE/HA/Makefile
@@ -1,5 +1,5 @@
 SOURCES=CRM.pm Env.pm Groups.pm Resources.pm Config.pm LRM.pm Manager.pm \
-	NodeStatus.pm Tools.pm FenceConfig.pm Hardware.pm
+	NodeStatus.pm Tools.pm FenceConfig.pm Hardware.pm Fence.pm
 
 .PHONY: install
 install:
-- 
2.1.4