[pve-devel] [RFC v2 pve-ha-manager 9/9] add tests for shutdown and restart behaviour
Thomas Lamprecht
t.lamprecht at proxmox.com
Mon Dec 14 15:30:01 CET 2015
Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
src/test/test-restart1/README | 2 ++
src/test/test-restart1/cmdlist | 4 +++
src/test/test-restart1/hardware_status | 5 ++++
src/test/test-restart1/log.expect | 48 ++++++++++++++++++++++++++++++
src/test/test-restart1/manager_status | 1 +
src/test/test-restart1/service_config | 5 ++++
src/test/test-shutdown1/README | 2 ++
src/test/test-shutdown1/cmdlist | 5 ++++
src/test/test-shutdown1/hardware_status | 5 ++++
src/test/test-shutdown1/log.expect | 52 +++++++++++++++++++++++++++++++++
src/test/test-shutdown1/manager_status | 1 +
src/test/test-shutdown1/service_config | 5 ++++
src/test/test-shutdown2/README | 2 ++
src/test/test-shutdown2/cmdlist | 5 ++++
src/test/test-shutdown2/hardware_status | 5 ++++
src/test/test-shutdown2/log.expect | 51 ++++++++++++++++++++++++++++++++
src/test/test-shutdown2/manager_status | 1 +
src/test/test-shutdown2/service_config | 5 ++++
18 files changed, 204 insertions(+)
create mode 100644 src/test/test-restart1/README
create mode 100644 src/test/test-restart1/cmdlist
create mode 100644 src/test/test-restart1/hardware_status
create mode 100644 src/test/test-restart1/log.expect
create mode 100644 src/test/test-restart1/manager_status
create mode 100644 src/test/test-restart1/service_config
create mode 100644 src/test/test-shutdown1/README
create mode 100644 src/test/test-shutdown1/cmdlist
create mode 100644 src/test/test-shutdown1/hardware_status
create mode 100644 src/test/test-shutdown1/log.expect
create mode 100644 src/test/test-shutdown1/manager_status
create mode 100644 src/test/test-shutdown1/service_config
create mode 100644 src/test/test-shutdown2/README
create mode 100644 src/test/test-shutdown2/cmdlist
create mode 100644 src/test/test-shutdown2/hardware_status
create mode 100644 src/test/test-shutdown2/log.expect
create mode 100644 src/test/test-shutdown2/manager_status
create mode 100644 src/test/test-shutdown2/service_config
diff --git a/src/test/test-restart1/README b/src/test/test-restart1/README
new file mode 100644
index 0000000..42102a2
--- /dev/null
+++ b/src/test/test-restart1/README
@@ -0,0 +1,2 @@
+Test graceful restart of a node, this should freeze
+the enabled services remaining on the node.
\ No newline at end of file
diff --git a/src/test/test-restart1/cmdlist b/src/test/test-restart1/cmdlist
new file mode 100644
index 0000000..70a401e
--- /dev/null
+++ b/src/test/test-restart1/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "power node3 restart" ]
+]
diff --git a/src/test/test-restart1/hardware_status b/src/test/test-restart1/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-restart1/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-restart1/log.expect b/src/test/test-restart1/log.expect
new file mode 100644
index 0000000..05498bb
--- /dev/null
+++ b/src/test/test-restart1/log.expect
@@ -0,0 +1,48 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:102': state changed from 'started' to 'request_stop'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 40 node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info 120 cmdlist: execute power node3 restart
+info 120 node3/lrm: reboot node
+notice 120 node3/lrm: restart LRM, freeze all services
+info 120 node1/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info 125 hardware: stopped LRM on 'node3'
+info 126 cmdhelper: execute power node3 off
+info 125 node3/crm: killed by poweroff
+info 140 node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info 142 cmdhelper: execute power node3 on
+info 140 node3/crm: status change startup => wait_for_quorum
+info 140 node3/lrm: status change startup => wait_for_agent_lock
+info 144 node3/crm: status change wait_for_quorum => slave
+info 160 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 160 node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info 165 node3/lrm: status change wait_for_agent_lock => active
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-restart1/manager_status b/src/test/test-restart1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-restart1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-restart1/service_config b/src/test/test-restart1/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-restart1/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/README b/src/test/test-shutdown1/README
new file mode 100644
index 0000000..ff56b5e
--- /dev/null
+++ b/src/test/test-shutdown1/README
@@ -0,0 +1,2 @@
+Test graceful shutdown of a node with no power on after that,
+this should trigger the fencing of the node.
\ No newline at end of file
diff --git a/src/test/test-shutdown1/cmdlist b/src/test/test-shutdown1/cmdlist
new file mode 100644
index 0000000..cf36b9a
--- /dev/null
+++ b/src/test/test-shutdown1/cmdlist
@@ -0,0 +1,5 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "delay 5" ],
+ [ "power node3 shutdown" ]
+]
diff --git a/src/test/test-shutdown1/hardware_status b/src/test/test-shutdown1/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-shutdown1/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/log.expect b/src/test/test-shutdown1/log.expect
new file mode 100644
index 0000000..0aee4e4
--- /dev/null
+++ b/src/test/test-shutdown1/log.expect
@@ -0,0 +1,52 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:102': state changed from 'started' to 'request_stop'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 40 node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info 220 cmdlist: execute power node3 shutdown
+notice 220 node3/lrm: stop LRM, remaining services will not get freezed
+info 220 node3/lrm: graceful shutdown
+info 220 node3/crm: graceful shutdown
+info 225 node3/lrm: stopping service vm:103
+info 225 node3/lrm: service status vm:103 stopped
+info 225 hardware: stopped LRM on 'node3'
+info 226 cmdhelper: execute power node3 off
+info 225 node3/crm: killed by poweroff
+info 240 node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info 280 node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info 280 node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+info 360 node1/crm: got lock 'ha_agent_node3_lock'
+info 360 node1/crm: fencing: acknowleged - got agent lock for node 'node3'
+info 360 node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+info 360 node1/crm: service 'vm:103': state changed from 'fence' to 'stopped'
+info 380 node1/crm: service 'vm:103': state changed from 'stopped' to 'started' (node = node2)
+info 383 node2/lrm: starting service vm:103
+info 383 node2/lrm: service status vm:103 started
+info 820 hardware: exit simulation - done
diff --git a/src/test/test-shutdown1/manager_status b/src/test/test-shutdown1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/service_config b/src/test/test-shutdown1/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown1/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/README b/src/test/test-shutdown2/README
new file mode 100644
index 0000000..57f111f
--- /dev/null
+++ b/src/test/test-shutdown2/README
@@ -0,0 +1,2 @@
+Test shutdown of a node with a power on shortly after that,
+this should NOT trigger fencing as the node comes up again fast.
\ No newline at end of file
diff --git a/src/test/test-shutdown2/cmdlist b/src/test/test-shutdown2/cmdlist
new file mode 100644
index 0000000..342845f
--- /dev/null
+++ b/src/test/test-shutdown2/cmdlist
@@ -0,0 +1,5 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on" ],
+ [ "delay 10" ],
+ [ "power node3 shutdown", "delay 20", "power node3 on" ]
+]
diff --git a/src/test/test-shutdown2/hardware_status b/src/test/test-shutdown2/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-shutdown2/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/log.expect b/src/test/test-shutdown2/log.expect
new file mode 100644
index 0000000..8a4894e
--- /dev/null
+++ b/src/test/test-shutdown2/log.expect
@@ -0,0 +1,51 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:102': state changed from 'started' to 'request_stop'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 40 node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info 220 cmdlist: execute power node3 shutdown
+notice 220 node3/lrm: stop LRM, remaining services will not get freezed
+info 220 node3/lrm: graceful shutdown
+info 220 node3/crm: graceful shutdown
+info 225 node3/lrm: stopping service vm:103
+info 225 node3/lrm: service status vm:103 stopped
+info 225 hardware: stopped LRM on 'node3'
+info 226 cmdhelper: execute power node3 off
+info 225 node3/crm: killed by poweroff
+info 240 node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info 242 cmdhelper: execute power node3 on
+info 240 node3/crm: status change startup => wait_for_quorum
+info 240 node3/lrm: status change startup => wait_for_agent_lock
+info 244 node3/crm: status change wait_for_quorum => slave
+info 245 node3/lrm: status change wait_for_agent_lock => active
+info 245 node3/lrm: starting service vm:103
+info 245 node3/lrm: service status vm:103 started
+info 260 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 820 hardware: exit simulation - done
diff --git a/src/test/test-shutdown2/manager_status b/src/test/test-shutdown2/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown2/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/service_config b/src/test/test-shutdown2/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown2/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
--
2.1.4
More information about the pve-devel
mailing list