[pve-devel] [RFC pve-ha-manager 8/8] add tests for shutdown and restart behaviour

Thomas Lamprecht t.lamprecht at proxmox.com
Fri Dec 11 20:01:45 CET 2015


Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/test/test-restart1/README                      |  2 +
 src/test/test-restart1/cmdlist                     |  4 ++
 src/test/test-restart1/hardware_status             |  5 +++
 src/test/test-restart1/log.expect                  | 48 +++++++++++++++++++++
 src/test/test-restart1/manager_status              |  1 +
 src/test/test-restart1/service_config              |  5 +++
 src/test/test-shutdown1/README                     |  2 +
 src/test/test-shutdown1/cmdlist                    |  5 +++
 src/test/test-shutdown1/hardware_status            |  5 +++
 src/test/test-shutdown1/log.expect                 | 50 ++++++++++++++++++++++
 src/test/test-shutdown1/manager_status             |  1 +
 src/test/test-shutdown1/service_config             |  5 +++
 src/test/test-shutdown2/README                     |  2 +
 src/test/test-shutdown2/cmdlist                    |  5 +++
 src/test/test-shutdown2/hardware_status            |  5 +++
 src/test/test-shutdown2/log.expect                 | 47 ++++++++++++++++++++
 src/test/test-shutdown2/manager_status             |  1 +
 src/test/test-shutdown2/service_config             |  5 +++
 src/test/test-shutdown2/status/cluster_locks       |  1 +
 src/test/test-shutdown2/status/crm_commands        |  0
 src/test/test-shutdown2/status/groups              | 11 +++++
 src/test/test-shutdown2/status/hardware.lck        |  0
 src/test/test-shutdown2/status/hardware_status     |  1 +
 src/test/test-shutdown2/status/log                 | 47 ++++++++++++++++++++
 src/test/test-shutdown2/status/lrm_status_node1    |  1 +
 src/test/test-shutdown2/status/lrm_status_node2    |  1 +
 src/test/test-shutdown2/status/lrm_status_node3    |  1 +
 src/test/test-shutdown2/status/manager_status      |  1 +
 src/test/test-shutdown2/status/service_config      |  5 +++
 .../test-shutdown2/status/service_status_node1     |  1 +
 .../test-shutdown2/status/service_status_node2     |  1 +
 .../test-shutdown2/status/service_status_node3     |  1 +
 src/test/test-shutdown2/status/watchdog_status     |  1 +
 33 files changed, 271 insertions(+)
 create mode 100644 src/test/test-restart1/README
 create mode 100644 src/test/test-restart1/cmdlist
 create mode 100644 src/test/test-restart1/hardware_status
 create mode 100644 src/test/test-restart1/log.expect
 create mode 100644 src/test/test-restart1/manager_status
 create mode 100644 src/test/test-restart1/service_config
 create mode 100644 src/test/test-shutdown1/README
 create mode 100644 src/test/test-shutdown1/cmdlist
 create mode 100644 src/test/test-shutdown1/hardware_status
 create mode 100644 src/test/test-shutdown1/log.expect
 create mode 100644 src/test/test-shutdown1/manager_status
 create mode 100644 src/test/test-shutdown1/service_config
 create mode 100644 src/test/test-shutdown2/README
 create mode 100644 src/test/test-shutdown2/cmdlist
 create mode 100644 src/test/test-shutdown2/hardware_status
 create mode 100644 src/test/test-shutdown2/log.expect
 create mode 100644 src/test/test-shutdown2/manager_status
 create mode 100644 src/test/test-shutdown2/service_config
 create mode 100644 src/test/test-shutdown2/status/cluster_locks
 create mode 100644 src/test/test-shutdown2/status/crm_commands
 create mode 100644 src/test/test-shutdown2/status/groups
 create mode 100644 src/test/test-shutdown2/status/hardware.lck
 create mode 100644 src/test/test-shutdown2/status/hardware_status
 create mode 100644 src/test/test-shutdown2/status/log
 create mode 100644 src/test/test-shutdown2/status/lrm_status_node1
 create mode 100644 src/test/test-shutdown2/status/lrm_status_node2
 create mode 100644 src/test/test-shutdown2/status/lrm_status_node3
 create mode 100644 src/test/test-shutdown2/status/manager_status
 create mode 100644 src/test/test-shutdown2/status/service_config
 create mode 100644 src/test/test-shutdown2/status/service_status_node1
 create mode 100644 src/test/test-shutdown2/status/service_status_node2
 create mode 100644 src/test/test-shutdown2/status/service_status_node3
 create mode 100644 src/test/test-shutdown2/status/watchdog_status

diff --git a/src/test/test-restart1/README b/src/test/test-restart1/README
new file mode 100644
index 0000000..42102a2
--- /dev/null
+++ b/src/test/test-restart1/README
@@ -0,0 +1,2 @@
+Test graceful restart of a node, this should freeze
+the enabled services remaining on the node.
\ No newline at end of file
diff --git a/src/test/test-restart1/cmdlist b/src/test/test-restart1/cmdlist
new file mode 100644
index 0000000..70a401e
--- /dev/null
+++ b/src/test/test-restart1/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "power node3 restart" ]
+]
diff --git a/src/test/test-restart1/hardware_status b/src/test/test-restart1/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-restart1/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-restart1/log.expect b/src/test/test-restart1/log.expect
new file mode 100644
index 0000000..05498bb
--- /dev/null
+++ b/src/test/test-restart1/log.expect
@@ -0,0 +1,48 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    120      cmdlist: execute power node3 restart
+info    120    node3/lrm: reboot node
+notice   120    node3/lrm: restart LRM, freeze all services
+info    120    node1/crm: service 'vm:103': state changed from 'started' to 'freeze' 
+info    125     hardware: stopped LRM on 'node3'
+info    126    cmdhelper: execute power node3 off
+info    125    node3/crm: killed by poweroff
+info    140    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    142    cmdhelper: execute power node3 on
+info    140    node3/crm: status change startup => wait_for_quorum
+info    140    node3/lrm: status change startup => wait_for_agent_lock
+info    144    node3/crm: status change wait_for_quorum => slave
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info    160    node1/crm: service 'vm:103': state changed from 'freeze' to 'started' 
+info    165    node3/lrm: status change wait_for_agent_lock => active
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-restart1/manager_status b/src/test/test-restart1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-restart1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-restart1/service_config b/src/test/test-restart1/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-restart1/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/README b/src/test/test-shutdown1/README
new file mode 100644
index 0000000..ff56b5e
--- /dev/null
+++ b/src/test/test-shutdown1/README
@@ -0,0 +1,2 @@
+Test graceful shutdown of a node with no power on after that,
+this should trigger the fencing of the node.
\ No newline at end of file
diff --git a/src/test/test-shutdown1/cmdlist b/src/test/test-shutdown1/cmdlist
new file mode 100644
index 0000000..cf36b9a
--- /dev/null
+++ b/src/test/test-shutdown1/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "delay 5" ],
+    [ "power node3 shutdown" ]
+]
diff --git a/src/test/test-shutdown1/hardware_status b/src/test/test-shutdown1/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-shutdown1/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/log.expect b/src/test/test-shutdown1/log.expect
new file mode 100644
index 0000000..861eebd
--- /dev/null
+++ b/src/test/test-shutdown1/log.expect
@@ -0,0 +1,50 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    220      cmdlist: execute power node3 shutdown
+notice   220    node3/lrm: stop LRM, remaining services will not get freezed
+info    220    node3/lrm: graceful shutdown
+info    220    node3/crm: graceful shutdown
+info    225     hardware: stopped LRM on 'node3'
+info    226    cmdhelper: execute power node3 off
+info    225    node3/crm: killed by poweroff
+info    240    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    280    node1/crm: service 'vm:103': state changed from 'started' to 'fence' 
+info    280    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+info    360    node1/crm: got lock 'ha_agent_node3_lock'
+info    360    node1/crm: fencing: acknowleged - got agent lock for node 'node3'
+info    360    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+info    360    node1/crm: service 'vm:103': state changed from 'fence' to 'stopped' 
+info    380    node1/crm: service 'vm:103': state changed from 'stopped' to 'started'  (node = node2)
+info    383    node2/lrm: starting service vm:103
+info    383    node2/lrm: service status vm:103 started
+info    820     hardware: exit simulation - done
diff --git a/src/test/test-shutdown1/manager_status b/src/test/test-shutdown1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown1/service_config b/src/test/test-shutdown1/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown1/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/README b/src/test/test-shutdown2/README
new file mode 100644
index 0000000..57f111f
--- /dev/null
+++ b/src/test/test-shutdown2/README
@@ -0,0 +1,2 @@
+Test shutdown of a node with a power on shortly after that,
+this should NOT trigger fencing as the node comes up again fast.
\ No newline at end of file
diff --git a/src/test/test-shutdown2/cmdlist b/src/test/test-shutdown2/cmdlist
new file mode 100644
index 0000000..342845f
--- /dev/null
+++ b/src/test/test-shutdown2/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "delay 10" ],
+    [ "power node3 shutdown", "delay 20", "power node3 on" ]
+]
diff --git a/src/test/test-shutdown2/hardware_status b/src/test/test-shutdown2/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-shutdown2/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/log.expect b/src/test/test-shutdown2/log.expect
new file mode 100644
index 0000000..a1cc68d
--- /dev/null
+++ b/src/test/test-shutdown2/log.expect
@@ -0,0 +1,47 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    220      cmdlist: execute power node3 shutdown
+notice   220    node3/lrm: stop LRM, remaining services will not get freezed
+info    220    node3/lrm: graceful shutdown
+info    220    node3/crm: graceful shutdown
+info    225     hardware: stopped LRM on 'node3'
+info    226    cmdhelper: execute power node3 off
+info    225    node3/crm: killed by poweroff
+info    240    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    242    cmdhelper: execute power node3 on
+info    240    node3/crm: status change startup => wait_for_quorum
+info    240    node3/lrm: status change startup => wait_for_agent_lock
+info    244    node3/crm: status change wait_for_quorum => slave
+info    245    node3/lrm: status change wait_for_agent_lock => active
+info    260    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info    820     hardware: exit simulation - done
diff --git a/src/test/test-shutdown2/manager_status b/src/test/test-shutdown2/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown2/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/service_config b/src/test/test-shutdown2/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown2/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/cluster_locks b/src/test/test-shutdown2/status/cluster_locks
new file mode 100644
index 0000000..a78b735
--- /dev/null
+++ b/src/test/test-shutdown2/status/cluster_locks
@@ -0,0 +1 @@
+{"ha_agent_node3_lock":{"node":"node3","time":805},"ha_agent_node1_lock":{"node":"node1","time":801},"ha_agent_node2_lock":{"node":"node2","time":803},"ha_manager_lock":{"time":800,"node":"node1"}}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/crm_commands b/src/test/test-shutdown2/status/crm_commands
new file mode 100644
index 0000000..e69de29
diff --git a/src/test/test-shutdown2/status/groups b/src/test/test-shutdown2/status/groups
new file mode 100644
index 0000000..e902563
--- /dev/null
+++ b/src/test/test-shutdown2/status/groups
@@ -0,0 +1,11 @@
+group: prefer_node1
+    nodes node1
+    nofailback 1
+
+group: prefer_node2
+    nodes node2
+    nofailback 1
+
+group: prefer_node3
+    nodes node3
+    nofailback 1
diff --git a/src/test/test-shutdown2/status/hardware.lck b/src/test/test-shutdown2/status/hardware.lck
new file mode 100644
index 0000000..e69de29
diff --git a/src/test/test-shutdown2/status/hardware_status b/src/test/test-shutdown2/status/hardware_status
new file mode 100644
index 0000000..bf4c51c
--- /dev/null
+++ b/src/test/test-shutdown2/status/hardware_status
@@ -0,0 +1 @@
+{"node2":{"network":"on","power":"on","currentaction":"on"},"node3":{"network":"on","power":"on","currentaction":"on"},"node1":{"currentaction":"on","power":"on","network":"on"}}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/log b/src/test/test-shutdown2/status/log
new file mode 100644
index 0000000..a1cc68d
--- /dev/null
+++ b/src/test/test-shutdown2/status/log
@@ -0,0 +1,47 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:102': state changed from 'started' to 'request_stop' 
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped' 
+info    220      cmdlist: execute power node3 shutdown
+notice   220    node3/lrm: stop LRM, remaining services will not get freezed
+info    220    node3/lrm: graceful shutdown
+info    220    node3/crm: graceful shutdown
+info    225     hardware: stopped LRM on 'node3'
+info    226    cmdhelper: execute power node3 off
+info    225    node3/crm: killed by poweroff
+info    240    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    242    cmdhelper: execute power node3 on
+info    240    node3/crm: status change startup => wait_for_quorum
+info    240    node3/lrm: status change startup => wait_for_agent_lock
+info    244    node3/crm: status change wait_for_quorum => slave
+info    245    node3/lrm: status change wait_for_agent_lock => active
+info    260    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info    820     hardware: exit simulation - done
diff --git a/src/test/test-shutdown2/status/lrm_status_node1 b/src/test/test-shutdown2/status/lrm_status_node1
new file mode 100644
index 0000000..c9821d7
--- /dev/null
+++ b/src/test/test-shutdown2/status/lrm_status_node1
@@ -0,0 +1 @@
+{"results":{"Y+7jxRmyfSByMgxEXnxFyw":{"state":"started","sid":"vm:101","exit_code":0}},"mode":"active","timestamp":801}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/lrm_status_node2 b/src/test/test-shutdown2/status/lrm_status_node2
new file mode 100644
index 0000000..575dd90
--- /dev/null
+++ b/src/test/test-shutdown2/status/lrm_status_node2
@@ -0,0 +1 @@
+{"results":{"0Dkv+ZgdS7ybNCY1twXtvw":{"exit_code":0,"sid":"vm:102","state":"stopped"}},"mode":"active","timestamp":803}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/lrm_status_node3 b/src/test/test-shutdown2/status/lrm_status_node3
new file mode 100644
index 0000000..25be8c8
--- /dev/null
+++ b/src/test/test-shutdown2/status/lrm_status_node3
@@ -0,0 +1 @@
+{"results":{"Ga6WOSXbSz8Rl3BBd0RtFQ":{"state":"started","sid":"vm:103","exit_code":0}},"mode":"active","timestamp":805}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/manager_status b/src/test/test-shutdown2/status/manager_status
new file mode 100644
index 0000000..8f82c7d
--- /dev/null
+++ b/src/test/test-shutdown2/status/manager_status
@@ -0,0 +1 @@
+{"service_status":{"vm:101":{"node":"node1","state":"started","uid":"Y+7jxRmyfSByMgxEXnxFyw"},"vm:102":{"node":"node2","state":"stopped","uid":"0Dkv+ZgdS7ybNCY1twXtvw"},"vm:103":{"uid":"Ga6WOSXbSz8Rl3BBd0RtFQ","node":"node3","state":"started"}},"node_status":{"node2":"online","node3":"online","node1":"online"},"relocate_trial":{"vm:101":0,"vm:103":0},"master_node":"node1","timestamp":800}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/service_config b/src/test/test-shutdown2/status/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-shutdown2/status/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/service_status_node1 b/src/test/test-shutdown2/status/service_status_node1
new file mode 100644
index 0000000..45452c8
--- /dev/null
+++ b/src/test/test-shutdown2/status/service_status_node1
@@ -0,0 +1 @@
+{"vm:101":1}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/service_status_node2 b/src/test/test-shutdown2/status/service_status_node2
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-shutdown2/status/service_status_node2
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/service_status_node3 b/src/test/test-shutdown2/status/service_status_node3
new file mode 100644
index 0000000..0d26fda
--- /dev/null
+++ b/src/test/test-shutdown2/status/service_status_node3
@@ -0,0 +1 @@
+{"vm:103":1}
\ No newline at end of file
diff --git a/src/test/test-shutdown2/status/watchdog_status b/src/test/test-shutdown2/status/watchdog_status
new file mode 100644
index 0000000..f85786b
--- /dev/null
+++ b/src/test/test-shutdown2/status/watchdog_status
@@ -0,0 +1 @@
+{"WD:node3:9711:5":{"node":"node3","update_time":805},"WD:node1:9711:1":{"node":"node1","update_time":800},"WD:node2:9711:3":{"node":"node2","update_time":803},"WD:node1:9711:2":{"node":"node1","update_time":801}}
\ No newline at end of file
-- 
2.1.4





More information about the pve-devel mailing list