[pve-devel] [RFC v2 pve-ha-manager 9/9] add tests for shutdown and restart behaviour

Thomas Lamprecht t.lamprecht at proxmox.com
Mon Dec 14 15:30:01 CET 2015


Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/test/test-restart1/README           |  2 ++
 src/test/test-restart1/cmdlist          |  4 +++
 src/test/test-restart1/hardware_status  |  5 ++++
 src/test/test-restart1/log.expect       | 48 ++++++++++++++++++++++++++++++
 src/test/test-restart1/manager_status   |  1 +
 src/test/test-restart1/service_config   |  5 ++++
 src/test/test-shutdown1/README          |  2 ++
 src/test/test-shutdown1/cmdlist         |  5 ++++
 src/test/test-shutdown1/hardware_status |  5 ++++
 src/test/test-shutdown1/log.expect      | 52 +++++++++++++++++++++++++++++++++
 src/test/test-shutdown1/manager_status  |  1 +
 src/test/test-shutdown1/service_config  |  5 ++++
 src/test/test-shutdown2/README          |  2 ++
 src/test/test-shutdown2/cmdlist         |  5 ++++
 src/test/test-shutdown2/hardware_status |  5 ++++
 src/test/test-shutdown2/log.expect      | 51 ++++++++++++++++++++++++++++++++
 src/test/test-shutdown2/manager_status  |  1 +
 src/test/test-shutdown2/service_config  |  5 ++++
 18 files changed, 204 insertions(+)
 create mode 100644 src/test/test-restart1/README
 create mode 100644 src/test/test-restart1/cmdlist
 create mode 100644 src/test/test-restart1/hardware_status
 create mode 100644 src/test/test-restart1/log.expect
 create mode 100644 src/test/test-restart1/manager_status
 create mode 100644 src/test/test-restart1/service_config
 create mode 100644 src/test/test-shutdown1/README
 create mode 100644 src/test/test-shutdown1/cmdlist
 create mode 100644 src/test/test-shutdown1/hardware_status
 create mode 100644 src/test/test-shutdown1/log.expect
 create mode 100644 src/test/test-shutdown1/manager_status
 create mode 100644 src/test/test-shutdown1/service_config
 create mode 100644 src/test/test-shutdown2/README
 create mode 100644 src/test/test-shutdown2/cmdlist
 create mode 100644 src/test/test-shutdown2/hardware_status
 create mode 100644 src/test/test-shutdown2/log.expect
 create mode 100644 src/test/test-shutdown2/manager_status
 create mode 100644 src/test/test-shutdown2/service_config

diff --git a/src/test/test-restart1/README b/src/test/test-restart1/README
new file mode 100644
index 0000000..42102a2
--- /dev/null
+++ b/src/test/test-restart1/README
@@ -0,0 +1,2 @@
+Test graceful restart of a node, this should freeze
+the enabled services remaining on the node.
\ No newline at end of file
diff --git a/src/test/test-restart1/cmdlist b/src/test/test-restart1/cmdlist
new file mode 100644
index 0000000..70a401e
--- /dev/null
+++ b/src/test/test-restart1/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "power node3 restart" ]
+]
diff --git a/src/test/test-restart1/hardware_status b/src/test/test-restart1/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-restart1/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-restart1/log.expect b/src/test/test-restart1/log.expect
new file mode 100644
index 0000000..05498bb
--- /dev/null
+++ b/src/test/test-restart1/log.expect
@@ -0,0 +1,48 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    120      cmdlist: execute power node3 restart
+info    120    node3/lrm: reboot node
+notice   120    node3/lrm: restart LRM, freeze all services
+info    120    node1/crm: service 'vm:103': state changed from 'started' to 'freeze' 
+info    125     hardware: stopped LRM on 'node3'
+info    126    cmdhelper: execute power node3 off
+info    125    node3/crm: killed by poweroff
+info    140    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    142    cmdhelper: execute power node3 on
+info    140    node3/crm: status change startup => wait_for_quorum
+info    140    node3/lrm: status change startup => wait_for_agent_lock
+info    144    node3/crm: status change wait_for_quorum => slave
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info    160    node1/crm: service 'vm:103': state changed from 'freeze' to 'started' 
+info    165    node3/lrm: status change wait_for_agent_lock => active
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-restart1/manager_status b/src/test/test-restart1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-restart1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-restart1/service_config b/src/test/test-restart1/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-restart1/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/README b/src/test/test-shutdown1/README
new file mode 100644
index 0000000..ff56b5e
--- /dev/null
+++ b/src/test/test-shutdown1/README
@@ -0,0 +1,2 @@
+Test graceful shutdown of a node with no power on after that,
+this should trigger the fencing of the node.
\ No newline at end of file
diff --git a/src/test/test-shutdown1/cmdlist b/src/test/test-shutdown1/cmdlist
new file mode 100644
index 0000000..cf36b9a
--- /dev/null
+++ b/src/test/test-shutdown1/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "delay 5" ],
+    [ "power node3 shutdown" ]
+]
diff --git a/src/test/test-shutdown1/hardware_status b/src/test/test-shutdown1/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-shutdown1/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/log.expect b/src/test/test-shutdown1/log.expect
new file mode 100644
index 0000000..0aee4e4
--- /dev/null
+++ b/src/test/test-shutdown1/log.expect
@@ -0,0 +1,52 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    220      cmdlist: execute power node3 shutdown
+notice   220    node3/lrm: stop LRM, remaining services will not get freezed
+info    220    node3/lrm: graceful shutdown
+info    220    node3/crm: graceful shutdown
+info    225    node3/lrm: stopping service vm:103
+info    225    node3/lrm: service status vm:103 stopped
+info    225     hardware: stopped LRM on 'node3'
+info    226    cmdhelper: execute power node3 off
+info    225    node3/crm: killed by poweroff
+info    240    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    280    node1/crm: service 'vm:103': state changed from 'started' to 'fence' 
+info    280    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+info    360    node1/crm: got lock 'ha_agent_node3_lock'
+info    360    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
+info    360    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+info    360    node1/crm: service 'vm:103': state changed from 'fence' to 'stopped' 
+info    380    node1/crm: service 'vm:103': state changed from 'stopped' to 'started'  (node = node2)
+info    383    node2/lrm: starting service vm:103
+info    383    node2/lrm: service status vm:103 started
+info    820     hardware: exit simulation - done
diff --git a/src/test/test-shutdown1/manager_status b/src/test/test-shutdown1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/service_config b/src/test/test-shutdown1/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown1/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/README b/src/test/test-shutdown2/README
new file mode 100644
index 0000000..57f111f
--- /dev/null
+++ b/src/test/test-shutdown2/README
@@ -0,0 +1,2 @@
+Test shutdown of a node with a power on shortly after that,
+this should NOT trigger fencing as the node comes up again fast.
\ No newline at end of file
diff --git a/src/test/test-shutdown2/cmdlist b/src/test/test-shutdown2/cmdlist
new file mode 100644
index 0000000..342845f
--- /dev/null
+++ b/src/test/test-shutdown2/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "delay 10" ],
+    [ "power node3 shutdown", "delay 20", "power node3 on" ]
+]
diff --git a/src/test/test-shutdown2/hardware_status b/src/test/test-shutdown2/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-shutdown2/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/log.expect b/src/test/test-shutdown2/log.expect
new file mode 100644
index 0000000..8a4894e
--- /dev/null
+++ b/src/test/test-shutdown2/log.expect
@@ -0,0 +1,51 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    220      cmdlist: execute power node3 shutdown
+notice   220    node3/lrm: stop LRM, remaining services will not get freezed
+info    220    node3/lrm: graceful shutdown
+info    220    node3/crm: graceful shutdown
+info    225    node3/lrm: stopping service vm:103
+info    225    node3/lrm: service status vm:103 stopped
+info    225     hardware: stopped LRM on 'node3'
+info    226    cmdhelper: execute power node3 off
+info    225    node3/crm: killed by poweroff
+info    240    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    242    cmdhelper: execute power node3 on
+info    240    node3/crm: status change startup => wait_for_quorum
+info    240    node3/lrm: status change startup => wait_for_agent_lock
+info    244    node3/crm: status change wait_for_quorum => slave
+info    245    node3/lrm: status change wait_for_agent_lock => active
+info    245    node3/lrm: starting service vm:103
+info    245    node3/lrm: service status vm:103 started
+info    260    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info    820     hardware: exit simulation - done
diff --git a/src/test/test-shutdown2/manager_status b/src/test/test-shutdown2/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown2/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/service_config b/src/test/test-shutdown2/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown2/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
-- 
2.1.4




More information about the pve-devel mailing list