[pve-devel] [PATCH ha-manager v2 5/7] send email on fence failure and success

[Thomas Lamprecht]

Fencing is something which should not happen often in the real world
and has most time a really bad cause, thus send a email when
starting to fence a node and on success to root at localhost to inform
the cluster admin of said failures so he can check the hardware and
cluster status as soon as possible.

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---

same as previously

 src/PVE/HA/NodeStatus.pm | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/PVE/HA/NodeStatus.pm b/src/PVE/HA/NodeStatus.pm
index d9ef912..632dbd4 100644
--- a/src/PVE/HA/NodeStatus.pm
+++ b/src/PVE/HA/NodeStatus.pm
@@ -3,6 +3,7 @@ package PVE::HA::NodeStatus;
 use strict;
 use warnings;
 
+use JSON;
 use Data::Dumper;
 
 my $fence_delay = 60;
@@ -169,6 +170,38 @@ sub update {
    }
 }
 
+# assembles a commont text for fence emails
+my $send_fence_state_email = sub {
+    my ($self, $subject_prefix, $subject, $node) = @_;
+
+    my $haenv = $self->{haenv};
+
+    my $mail_text = <<EOF
+The node '$node' failed and needs manual intervention.
+
+The PVE HA manager tries  to fence it and recover the
+configured HA resources to a healthy node if possible.
+
+Current fence status:  $subject_prefix
+$subject
+
+
+Overall Cluster status:
+-----------------------
+
+EOF
+;
+    my $mail_subject = $subject_prefix . ': ' . $subject;
+
+    my $status = $haenv->read_manager_status();
+    my $data = { manager_status => $status, node_status => $self->{status} };
+
+    $mail_text .= to_json($data, { pretty => 1, canonical => 1});
+
+    $haenv->sendmail($mail_subject, $mail_text);
+};
+
+
 # start fencing
 sub fence_node {
     my ($self, $node) = @_;
@@ -179,13 +212,17 @@ sub fence_node {
 
     if ($state ne 'fence') {
 	&$set_node_state($self, $node, 'fence');
+	my $msg = "Try to fence node '$node'";
+	&$send_fence_state_email($self, 'FENCE', $msg, $node);
     }
 
     my $success = $haenv->get_ha_agent_lock($node);
 
     if ($success) {
-	$haenv->log("info", "fencing: acknowleged - got agent lock for node '$node'");
+	my $msg = "fencing: acknowleged - got agent lock for node '$node'";
+	$haenv->log("info", $msg);
 	&$set_node_state($self, $node, 'unknown');
+	&$send_fence_state_email($self, 'SUCEED', $msg, $node);
     }
 
     return $success;
-- 
2.1.4