[pve-devel] [RFC ha-manager 5/5] send email on fence failure and success

Thomas Lamprecht t.lamprecht at proxmox.com
Fri Apr 8 16:51:21 CEST 2016


Send a email when starting to fence a node and on either success or
failure.

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/PVE/HA/NodeStatus.pm           | 43 +++++++++++++++++++++++++++++++++++---
 src/test/test-basic1/log.expect    |  2 ++
 src/test/test-basic2/log.expect    |  1 +
 src/test/test-basic5/log.expect    |  2 ++
 src/test/test-hw-fence1/log.expect |  2 ++
 src/test/test-shutdown1/log.expect |  2 ++
 src/test/test-shutdown2/log.expect |  2 ++
 src/test/test-shutdown3/log.expect |  2 ++
 src/test/test-shutdown4/log.expect |  2 ++
 9 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/src/PVE/HA/NodeStatus.pm b/src/PVE/HA/NodeStatus.pm
index 6b29f9c..aecb0c8 100644
--- a/src/PVE/HA/NodeStatus.pm
+++ b/src/PVE/HA/NodeStatus.pm
@@ -2,6 +2,9 @@ package PVE::HA::NodeStatus;
 
 use strict;
 use warnings;
+
+use JSON;
+
 use PVE::HA::Fence;
 
 use Data::Dumper;
@@ -166,6 +169,37 @@ sub update {
    }
 }
 
+# assembles a commont text for fence emails
+my $send_fence_state_email = sub {
+    my ($self, $subject_prefix, $subject, $node) = @_;
+
+    my $haenv = $self->{haenv};
+
+    my $mail_text = <<EOF
+The node '$node' failed and needs manual intervention.
+
+The PVE HA manager tries  to fence it and recover the
+configured HA resources to a healthy node if possible.
+
+Current fence status:  $subject_prefix
+$subject
+
+
+Overall Cluster status:
+-----------------------
+
+EOF
+;
+    my $mail_subject = $subject_prefix . ': ' . $subject;
+
+    my $status = $haenv->read_manager_status();
+    my $data = { manager_status => $status, node_status => $self->{status} };
+
+    $mail_text .= to_json($data, { pretty => 1, canonical => 1});
+
+    $haenv->sendmail($mail_subject, $mail_text);
+};
+
 # start fencing
 sub fence_node {
     my ($self, $node) = @_;
@@ -188,8 +222,9 @@ sub fence_node {
 
 	# bad fence.cfg or no devices and only hardware fencing configured
 	if ($hw_fence_success < 0 && $fencing_mode eq 'hardware') {
-	    $haenv->log('err', "Fencing of node '$node' failed and needs " .
-			"manual intervention!");
+	    my $msg = "Fencing of node '$node' failed and needs manual intervention!";
+	    $haenv->log('err', $msg);
+	    &$send_fence_state_email($self, 'FAILED', $msg, $node);
 	    return 0;
 	}
 
@@ -216,8 +251,10 @@ sub fence_node {
     }
 
     if ($success) {
-	$haenv->log("info", "fencing: acknowleged - got agent lock for node '$node'");
+	my $msg = "fencing: acknowleged - got agent lock for node '$node'";
+	$haenv->log("info", $msg);
 	&$set_node_state($self, $node, 'unknown');
+	&$send_fence_state_email($self, 'SUCEED', $msg, $node);
 	PVE::HA::Fence::kill_and_cleanup_jobs($node) if ($fencing_mode ne 'watchdog');
     }
 
diff --git a/src/test/test-basic1/log.expect b/src/test/test-basic1/log.expect
index 68df71b..250b918 100644
--- a/src/test/test-basic1/log.expect
+++ b/src/test/test-basic1/log.expect
@@ -36,6 +36,7 @@ info    124    node3/crm: status change slave => wait_for_quorum
 info    125    node3/lrm: status change active => lost_agent_lock
 info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
 info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
 info    166     watchdog: execute power node3 off
 info    165    node3/crm: killed by poweroff
 info    166    node3/lrm: killed by poweroff
@@ -43,6 +44,7 @@ info    166     hardware: server 'node3' stopped by poweroff (watchdog)
 info    240    node1/crm: got lock 'ha_agent_node3_lock'
 info    240    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
 info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node3'
 info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node2'
 info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node2)
 info    243    node2/lrm: starting service vm:103
diff --git a/src/test/test-basic2/log.expect b/src/test/test-basic2/log.expect
index 72822ce..f20d09c 100644
--- a/src/test/test-basic2/log.expect
+++ b/src/test/test-basic2/log.expect
@@ -11,6 +11,7 @@ info     22    node3/crm: node 'node2': state changed from 'online' => 'unknown'
 info     22    node3/crm: got lock 'ha_agent_node1_lock'
 info     22    node3/crm: fencing: acknowleged - got agent lock for node 'node1'
 info     22    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai     22    node3/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node1'
 info     22    node3/crm: recover service 'vm:101' from fenced node 'node1' to node 'node3'
 info     22    node3/crm: service 'vm:101': state changed from 'fence' to 'started'  (node = node3)
 info     23    node3/lrm: got lock 'ha_agent_node3_lock'
diff --git a/src/test/test-basic5/log.expect b/src/test/test-basic5/log.expect
index 54b579c..8131797 100644
--- a/src/test/test-basic5/log.expect
+++ b/src/test/test-basic5/log.expect
@@ -43,9 +43,11 @@ info    222    node3/crm: status change slave => master
 info    222    node3/crm: node 'node1': state changed from 'online' => 'unknown'
 info    282    node3/crm: service 'vm:101': state changed from 'started' to 'fence'
 info    282    node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai    282    node3/crm: FENCE: Try to fence node 'node1'
 info    282    node3/crm: got lock 'ha_agent_node1_lock'
 info    282    node3/crm: fencing: acknowleged - got agent lock for node 'node1'
 info    282    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai    282    node3/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node1'
 info    282    node3/crm: recover service 'vm:101' from fenced node 'node1' to node 'node2'
 info    282    node3/crm: service 'vm:101': state changed from 'fence' to 'started'  (node = node2)
 info    301    node2/lrm: starting service vm:101
diff --git a/src/test/test-hw-fence1/log.expect b/src/test/test-hw-fence1/log.expect
index 15555d7..cb0b7a6 100644
--- a/src/test/test-hw-fence1/log.expect
+++ b/src/test/test-hw-fence1/log.expect
@@ -36,6 +36,7 @@ info    124    node3/crm: status change slave => wait_for_quorum
 info    125    node3/lrm: status change active => lost_agent_lock
 info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
 info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
 noti    160    node1/crm: Start fencing node 'node3'
 noti    160    node1/crm: [fence 'node3'] execute cmd: fence_virt --ip=127.0.0.1 --plug=102
 info    160   fence_virt: execute power node3 off
@@ -46,6 +47,7 @@ noti    180    node1/crm: fencing of node 'node3' succeeded, trying to get its a
 info    180    node1/crm: got lock 'ha_agent_node3_lock'
 info    180    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
 info    180    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    180    node1/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node3'
 info    180    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node2'
 info    180    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node2)
 info    183    node2/lrm: starting service vm:103
diff --git a/src/test/test-shutdown1/log.expect b/src/test/test-shutdown1/log.expect
index 9dbdb84..185e4ff 100644
--- a/src/test/test-shutdown1/log.expect
+++ b/src/test/test-shutdown1/log.expect
@@ -31,9 +31,11 @@ info    145     shutdown: execute power node3 off
 info    160    node1/crm: node 'node3': state changed from 'online' => 'unknown'
 info    200    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
 info    200    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    200    node1/crm: FENCE: Try to fence node 'node3'
 info    200    node1/crm: got lock 'ha_agent_node3_lock'
 info    200    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
 info    200    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    200    node1/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node3'
 info    200    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
 info    200    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
 info    201    node1/lrm: got lock 'ha_agent_node1_lock'
diff --git a/src/test/test-shutdown2/log.expect b/src/test/test-shutdown2/log.expect
index c3fbb07..719bff6 100644
--- a/src/test/test-shutdown2/log.expect
+++ b/src/test/test-shutdown2/log.expect
@@ -31,9 +31,11 @@ info    145     shutdown: execute power node3 off
 info    160    node1/crm: node 'node3': state changed from 'online' => 'unknown'
 info    200    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
 info    200    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    200    node1/crm: FENCE: Try to fence node 'node3'
 info    200    node1/crm: got lock 'ha_agent_node3_lock'
 info    200    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
 info    200    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    200    node1/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node3'
 info    200    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
 info    200    node1/crm: service 'vm:103': state changed from 'fence' to 'started'  (node = node1)
 info    201    node1/lrm: got lock 'ha_agent_node1_lock'
diff --git a/src/test/test-shutdown3/log.expect b/src/test/test-shutdown3/log.expect
index 16d8c4e..6822925 100644
--- a/src/test/test-shutdown3/log.expect
+++ b/src/test/test-shutdown3/log.expect
@@ -31,9 +31,11 @@ info    145     shutdown: execute power node3 off
 info    160    node1/crm: node 'node3': state changed from 'online' => 'unknown'
 info    200    node1/crm: service 'ct:103': state changed from 'started' to 'fence'
 info    200    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    200    node1/crm: FENCE: Try to fence node 'node3'
 info    200    node1/crm: got lock 'ha_agent_node3_lock'
 info    200    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
 info    200    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    200    node1/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node3'
 info    200    node1/crm: recover service 'ct:103' from fenced node 'node3' to node 'node1'
 info    200    node1/crm: service 'ct:103': state changed from 'fence' to 'started'  (node = node1)
 info    201    node1/lrm: got lock 'ha_agent_node1_lock'
diff --git a/src/test/test-shutdown4/log.expect b/src/test/test-shutdown4/log.expect
index 843104b..47bc700 100644
--- a/src/test/test-shutdown4/log.expect
+++ b/src/test/test-shutdown4/log.expect
@@ -34,9 +34,11 @@ info    141    node2/crm: status change slave => master
 info    141    node2/crm: node 'node1': state changed from 'online' => 'unknown'
 info    220    node2/crm: service 'vm:100': state changed from 'started' to 'fence'
 info    220    node2/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai    220    node2/crm: FENCE: Try to fence node 'node1'
 info    220    node2/crm: got lock 'ha_agent_node1_lock'
 info    220    node2/crm: fencing: acknowleged - got agent lock for node 'node1'
 info    220    node2/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai    220    node2/crm: SUCEED: fencing: acknowleged - got agent lock for node 'node1'
 info    220    node2/crm: recover service 'vm:100' from fenced node 'node1' to node 'node2'
 info    220    node2/crm: service 'vm:100': state changed from 'fence' to 'started'  (node = node2)
 info    221    node2/lrm: got lock 'ha_agent_node2_lock'
-- 
2.1.4




More information about the pve-devel mailing list