[pve-devel] [RFC v3 pve-ha-manager 8/8] implement restart-lrm test

[Dietmar Maurer]

Signed-off-by: Dietmar Maurer <dietmar at proxmox.com>
---
 src/PVE/HA/Sim/TestHardware.pm            | 15 +++++++++++++--
 src/test/test-restart-lrm/cmdlist         |  4 ++++
 src/test/test-restart-lrm/hardware_status |  5 +++++
 src/test/test-restart-lrm/log.expect      | 30 ++++++++++++++++++++++++++++++
 src/test/test-restart-lrm/manager_status  |  1 +
 src/test/test-restart-lrm/service_config  |  3 +++
 6 files changed, 56 insertions(+), 2 deletions(-)
 create mode 100644 src/test/test-restart-lrm/cmdlist
 create mode 100644 src/test/test-restart-lrm/hardware_status
 create mode 100644 src/test/test-restart-lrm/log.expect
 create mode 100644 src/test/test-restart-lrm/manager_status
 create mode 100644 src/test/test-restart-lrm/service_config

diff --git a/src/PVE/HA/Sim/TestHardware.pm b/src/PVE/HA/Sim/TestHardware.pm
index 89af119..c0ac062 100644
--- a/src/PVE/HA/Sim/TestHardware.pm
+++ b/src/PVE/HA/Sim/TestHardware.pm
@@ -86,6 +86,7 @@ sub log {
 # network <node> <on|off>
 # reboot <node>
 # shutdown <node>
+# restart-lrm <node>
 
 sub sim_hardware_cmd {
     my ($self, $cmdstr, $logid) = @_;
@@ -109,6 +110,7 @@ sub sim_hardware_cmd {
 		if ($action eq 'on') {
 		    $d->{crm} = PVE::HA::CRM->new($d->{crm_env}) if !$d->{crm};
 		    $d->{lrm} = PVE::HA::LRM->new($d->{lrm_env}) if !$d->{lrm};
+		    $d->{lrm_restart} = undef;
 		} else {
 		    if ($d->{crm}) {
 			$d->{crm_env}->log('info', "killed by poweroff");
@@ -117,6 +119,7 @@ sub sim_hardware_cmd {
 		    if ($d->{lrm}) {
 			$d->{lrm_env}->log('info', "killed by poweroff");
 			$d->{lrm} = undef;
+			$d->{lrm_restart} = undef;
 		    }
 		    $self->watchdog_reset_nolock($node);
 		    $self->write_service_status($node, {});
@@ -141,7 +144,11 @@ sub sim_hardware_cmd {
 	    $self->write_hardware_status_nolock($cstatus);
 
 	    $d->{lrm}->shutdown_request() if $d->{lrm};
-
+	} elsif ($cmd eq 'restart-lrm') {
+	    if ($d->{lrm}) {
+		$d->{lrm_restart} = 1;
+		$d->{lrm}->shutdown_request();
+	    }
 	} else {
 	    die "sim_hardware_cmd: unknown command '$cmdstr'\n";
 	}
@@ -204,7 +211,11 @@ sub run {
 		    my $cstatus = $self->read_hardware_status_nolock();
 		    my $nstatus = $cstatus->{$node} || die "no node status for node '$node'";
 		    my $shutdown = $nstatus->{shutdown};
-		    if ($shutdown eq 'reboot') {
+		    if ($d->{lrm_restart}) {
+			die "lrm restart durin shutdown - not implemented" if $shutdown;
+			$d->{lrm_restart} = undef;
+			$d->{lrm} = PVE::HA::LRM->new($d->{lrm_env});
+		    } elsif ($shutdown eq 'reboot') {
 			$self->sim_hardware_cmd("power $node off", 'reboot');
 			$self->sim_hardware_cmd("power $node on", 'reboot');
 		    } elsif ($shutdown eq 'shutdown') {
diff --git a/src/test/test-restart-lrm/cmdlist b/src/test/test-restart-lrm/cmdlist
new file mode 100644
index 0000000..d58f644
--- /dev/null
+++ b/src/test/test-restart-lrm/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "restart-lrm node3" ]
+]
diff --git a/src/test/test-restart-lrm/hardware_status b/src/test/test-restart-lrm/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-restart-lrm/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-restart-lrm/log.expect b/src/test/test-restart-lrm/log.expect
new file mode 100644
index 0000000..88cbd5a
--- /dev/null
+++ b/src/test/test-restart-lrm/log.expect
@@ -0,0 +1,30 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    120      cmdlist: execute restart-lrm node3
+info    120    node3/lrm: restart LRM, freeze all services
+info    120    node1/crm: service 'vm:103': state changed from 'started' to 'freeze' 
+info    126    node3/lrm: exit (loop end)
+info    126    node3/lrm: status change startup => wait_for_agent_lock
+info    160    node1/crm: service 'vm:103': state changed from 'freeze' to 'started' 
+info    165    node3/lrm: status change wait_for_agent_lock => active
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-restart-lrm/manager_status b/src/test/test-restart-lrm/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-restart-lrm/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-restart-lrm/service_config b/src/test/test-restart-lrm/service_config
new file mode 100644
index 0000000..c3b22c8
--- /dev/null
+++ b/src/test/test-restart-lrm/service_config
@@ -0,0 +1,3 @@
+{
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
-- 
2.1.4