[pve-devel] [PATCH v2 ha-manager 14/15] test: add tests for static resource scheduling
Fiona Ebner
f.ebner at proxmox.com
Thu Nov 17 15:00:15 CET 2022
See the READMEs for more information about the tests.
Signed-off-by: Fiona Ebner <f.ebner at proxmox.com>
---
New in v2.
src/test/test-crs-static1/README | 4 +
src/test/test-crs-static1/cmdlist | 4 +
src/test/test-crs-static1/datacenter.cfg | 6 +
src/test/test-crs-static1/hardware_status | 5 +
src/test/test-crs-static1/log.expect | 50 +++++
src/test/test-crs-static1/manager_status | 1 +
src/test/test-crs-static1/service_config | 3 +
.../test-crs-static1/static_service_stats | 3 +
src/test/test-crs-static2/README | 4 +
src/test/test-crs-static2/cmdlist | 20 ++
src/test/test-crs-static2/datacenter.cfg | 6 +
src/test/test-crs-static2/groups | 2 +
src/test/test-crs-static2/hardware_status | 7 +
src/test/test-crs-static2/log.expect | 171 ++++++++++++++++++
src/test/test-crs-static2/manager_status | 1 +
src/test/test-crs-static2/service_config | 3 +
.../test-crs-static2/static_service_stats | 3 +
src/test/test-crs-static3/README | 5 +
src/test/test-crs-static3/cmdlist | 4 +
src/test/test-crs-static3/datacenter.cfg | 9 +
src/test/test-crs-static3/hardware_status | 5 +
src/test/test-crs-static3/log.expect | 131 ++++++++++++++
src/test/test-crs-static3/manager_status | 1 +
src/test/test-crs-static3/service_config | 12 ++
.../test-crs-static3/static_service_stats | 12 ++
src/test/test-crs-static4/README | 6 +
src/test/test-crs-static4/cmdlist | 4 +
src/test/test-crs-static4/datacenter.cfg | 9 +
src/test/test-crs-static4/hardware_status | 5 +
src/test/test-crs-static4/log.expect | 149 +++++++++++++++
src/test/test-crs-static4/manager_status | 1 +
src/test/test-crs-static4/service_config | 12 ++
.../test-crs-static4/static_service_stats | 12 ++
src/test/test-crs-static5/README | 5 +
src/test/test-crs-static5/cmdlist | 4 +
src/test/test-crs-static5/datacenter.cfg | 9 +
src/test/test-crs-static5/hardware_status | 5 +
src/test/test-crs-static5/log.expect | 117 ++++++++++++
src/test/test-crs-static5/manager_status | 1 +
src/test/test-crs-static5/service_config | 10 +
.../test-crs-static5/static_service_stats | 11 ++
41 files changed, 832 insertions(+)
create mode 100644 src/test/test-crs-static1/README
create mode 100644 src/test/test-crs-static1/cmdlist
create mode 100644 src/test/test-crs-static1/datacenter.cfg
create mode 100644 src/test/test-crs-static1/hardware_status
create mode 100644 src/test/test-crs-static1/log.expect
create mode 100644 src/test/test-crs-static1/manager_status
create mode 100644 src/test/test-crs-static1/service_config
create mode 100644 src/test/test-crs-static1/static_service_stats
create mode 100644 src/test/test-crs-static2/README
create mode 100644 src/test/test-crs-static2/cmdlist
create mode 100644 src/test/test-crs-static2/datacenter.cfg
create mode 100644 src/test/test-crs-static2/groups
create mode 100644 src/test/test-crs-static2/hardware_status
create mode 100644 src/test/test-crs-static2/log.expect
create mode 100644 src/test/test-crs-static2/manager_status
create mode 100644 src/test/test-crs-static2/service_config
create mode 100644 src/test/test-crs-static2/static_service_stats
create mode 100644 src/test/test-crs-static3/README
create mode 100644 src/test/test-crs-static3/cmdlist
create mode 100644 src/test/test-crs-static3/datacenter.cfg
create mode 100644 src/test/test-crs-static3/hardware_status
create mode 100644 src/test/test-crs-static3/log.expect
create mode 100644 src/test/test-crs-static3/manager_status
create mode 100644 src/test/test-crs-static3/service_config
create mode 100644 src/test/test-crs-static3/static_service_stats
create mode 100644 src/test/test-crs-static4/README
create mode 100644 src/test/test-crs-static4/cmdlist
create mode 100644 src/test/test-crs-static4/datacenter.cfg
create mode 100644 src/test/test-crs-static4/hardware_status
create mode 100644 src/test/test-crs-static4/log.expect
create mode 100644 src/test/test-crs-static4/manager_status
create mode 100644 src/test/test-crs-static4/service_config
create mode 100644 src/test/test-crs-static4/static_service_stats
create mode 100644 src/test/test-crs-static5/README
create mode 100644 src/test/test-crs-static5/cmdlist
create mode 100644 src/test/test-crs-static5/datacenter.cfg
create mode 100644 src/test/test-crs-static5/hardware_status
create mode 100644 src/test/test-crs-static5/log.expect
create mode 100644 src/test/test-crs-static5/manager_status
create mode 100644 src/test/test-crs-static5/service_config
create mode 100644 src/test/test-crs-static5/static_service_stats
diff --git a/src/test/test-crs-static1/README b/src/test/test-crs-static1/README
new file mode 100644
index 0000000..483f265
--- /dev/null
+++ b/src/test/test-crs-static1/README
@@ -0,0 +1,4 @@
+Test how service recovery works with the 'static' resource scheduling mode.
+
+Expect that the single service gets recovered to the node with the most
+available resources.
diff --git a/src/test/test-crs-static1/cmdlist b/src/test/test-crs-static1/cmdlist
new file mode 100644
index 0000000..8684073
--- /dev/null
+++ b/src/test/test-crs-static1/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "network node1 off" ]
+]
diff --git a/src/test/test-crs-static1/datacenter.cfg b/src/test/test-crs-static1/datacenter.cfg
new file mode 100644
index 0000000..8f83457
--- /dev/null
+++ b/src/test/test-crs-static1/datacenter.cfg
@@ -0,0 +1,6 @@
+{
+ "crs": {
+ "ha": "static"
+ }
+}
+
diff --git a/src/test/test-crs-static1/hardware_status b/src/test/test-crs-static1/hardware_status
new file mode 100644
index 0000000..0fa8c26
--- /dev/null
+++ b/src/test/test-crs-static1/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 200000000000 },
+ "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 300000000000 }
+}
diff --git a/src/test/test-crs-static1/log.expect b/src/test/test-crs-static1/log.expect
new file mode 100644
index 0000000..2b06b3c
--- /dev/null
+++ b/src/test/test-crs-static1/log.expect
@@ -0,0 +1,50 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: using scheduler mode 'static'
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:102' on node 'node1'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:102
+info 21 node1/lrm: service status vm:102 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 24 node3/crm: status change wait_for_quorum => slave
+info 120 cmdlist: execute network node1 off
+info 120 node1/crm: status change master => lost_manager_lock
+info 120 node1/crm: status change lost_manager_lock => wait_for_quorum
+info 121 node1/lrm: status change active => lost_agent_lock
+info 162 watchdog: execute power node1 off
+info 161 node1/crm: killed by poweroff
+info 162 node1/lrm: killed by poweroff
+info 162 hardware: server 'node1' stopped by poweroff (watchdog)
+info 222 node3/crm: got lock 'ha_manager_lock'
+info 222 node3/crm: status change slave => master
+info 222 node3/crm: using scheduler mode 'static'
+info 222 node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info 282 node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info 282 node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai 282 node3/crm: FENCE: Try to fence node 'node1'
+info 282 node3/crm: got lock 'ha_agent_node1_lock'
+info 282 node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info 282 node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai 282 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info 282 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info 282 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node3'
+info 282 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node3)
+info 283 node3/lrm: got lock 'ha_agent_node3_lock'
+info 283 node3/lrm: status change wait_for_agent_lock => active
+info 283 node3/lrm: starting service vm:102
+info 283 node3/lrm: service status vm:102 started
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-crs-static1/manager_status b/src/test/test-crs-static1/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static1/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static1/service_config b/src/test/test-crs-static1/service_config
new file mode 100644
index 0000000..9c12447
--- /dev/null
+++ b/src/test/test-crs-static1/service_config
@@ -0,0 +1,3 @@
+{
+ "vm:102": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static1/static_service_stats b/src/test/test-crs-static1/static_service_stats
new file mode 100644
index 0000000..7fb992d
--- /dev/null
+++ b/src/test/test-crs-static1/static_service_stats
@@ -0,0 +1,3 @@
+{
+ "vm:102": { "maxcpu": 2, "maxmem": 4000000000 }
+}
diff --git a/src/test/test-crs-static2/README b/src/test/test-crs-static2/README
new file mode 100644
index 0000000..61530a7
--- /dev/null
+++ b/src/test/test-crs-static2/README
@@ -0,0 +1,4 @@
+Test how service recovery works with the 'static' resource scheduling mode.
+
+Expect that the single service always gets recovered to the node with the most
+available resources. Also tests that the group priority still takes precedence.
diff --git a/src/test/test-crs-static2/cmdlist b/src/test/test-crs-static2/cmdlist
new file mode 100644
index 0000000..bada1bb
--- /dev/null
+++ b/src/test/test-crs-static2/cmdlist
@@ -0,0 +1,20 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on", "power node5 on" ],
+ [ "power node1 off" ],
+ [ "delay 300" ],
+ [ "power node1 on" ],
+ [ "delay 300" ],
+ [ "power node4 on" ],
+ [ "power node1 off" ],
+ [ "delay 300" ],
+ [ "power node1 on" ],
+ [ "delay 300" ],
+ [ "power node2 off" ],
+ [ "power node1 off" ],
+ [ "delay 300" ],
+ [ "power node1 on" ],
+ [ "delay 300" ],
+ [ "power node2 on" ],
+ [ "power node3 off" ],
+ [ "power node1 off" ]
+]
diff --git a/src/test/test-crs-static2/datacenter.cfg b/src/test/test-crs-static2/datacenter.cfg
new file mode 100644
index 0000000..8f83457
--- /dev/null
+++ b/src/test/test-crs-static2/datacenter.cfg
@@ -0,0 +1,6 @@
+{
+ "crs": {
+ "ha": "static"
+ }
+}
+
diff --git a/src/test/test-crs-static2/groups b/src/test/test-crs-static2/groups
new file mode 100644
index 0000000..43e9bf5
--- /dev/null
+++ b/src/test/test-crs-static2/groups
@@ -0,0 +1,2 @@
+group: prefer_node1
+ nodes node1
diff --git a/src/test/test-crs-static2/hardware_status b/src/test/test-crs-static2/hardware_status
new file mode 100644
index 0000000..d426023
--- /dev/null
+++ b/src/test/test-crs-static2/hardware_status
@@ -0,0 +1,7 @@
+{
+ "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 200000000000 },
+ "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 300000000000 },
+ "node4": { "power": "off", "network": "off", "cpus": 64, "memory": 300000000000 },
+ "node5": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static2/log.expect b/src/test/test-crs-static2/log.expect
new file mode 100644
index 0000000..ee4416c
--- /dev/null
+++ b/src/test/test-crs-static2/log.expect
@@ -0,0 +1,171 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node5 on
+info 20 node5/crm: status change startup => wait_for_quorum
+info 20 node5/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: using scheduler mode 'static'
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node5': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:102' on node 'node1'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:102
+info 21 node1/lrm: service status vm:102 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 24 node3/crm: status change wait_for_quorum => slave
+info 26 node5/crm: status change wait_for_quorum => slave
+info 120 cmdlist: execute power node1 off
+info 120 node1/crm: killed by poweroff
+info 120 node1/lrm: killed by poweroff
+info 220 cmdlist: execute delay 300
+info 222 node3/crm: got lock 'ha_manager_lock'
+info 222 node3/crm: status change slave => master
+info 222 node3/crm: using scheduler mode 'static'
+info 222 node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info 282 node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info 282 node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai 282 node3/crm: FENCE: Try to fence node 'node1'
+info 282 node3/crm: got lock 'ha_agent_node1_lock'
+info 282 node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info 282 node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai 282 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info 282 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info 282 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node3'
+info 282 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node3)
+info 283 node3/lrm: got lock 'ha_agent_node3_lock'
+info 283 node3/lrm: status change wait_for_agent_lock => active
+info 283 node3/lrm: starting service vm:102
+info 283 node3/lrm: service status vm:102 started
+info 600 cmdlist: execute power node1 on
+info 600 node1/crm: status change startup => wait_for_quorum
+info 600 node1/lrm: status change startup => wait_for_agent_lock
+info 600 node1/crm: status change wait_for_quorum => slave
+info 604 node3/crm: node 'node1': state changed from 'unknown' => 'online'
+info 604 node3/crm: migrate service 'vm:102' to node 'node1' (running)
+info 604 node3/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node3, target = node1)
+info 605 node3/lrm: service vm:102 - start migrate to node 'node1'
+info 605 node3/lrm: service vm:102 - end migrate to node 'node1'
+info 624 node3/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1)
+info 641 node1/lrm: got lock 'ha_agent_node1_lock'
+info 641 node1/lrm: status change wait_for_agent_lock => active
+info 641 node1/lrm: starting service vm:102
+info 641 node1/lrm: service status vm:102 started
+info 700 cmdlist: execute delay 300
+info 1080 cmdlist: execute power node4 on
+info 1080 node4/crm: status change startup => wait_for_quorum
+info 1080 node4/lrm: status change startup => wait_for_agent_lock
+info 1084 node3/crm: node 'node4': state changed from 'unknown' => 'online'
+info 1086 node4/crm: status change wait_for_quorum => slave
+info 1180 cmdlist: execute power node1 off
+info 1180 node1/crm: killed by poweroff
+info 1180 node1/lrm: killed by poweroff
+info 1182 node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info 1242 node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info 1242 node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai 1242 node3/crm: FENCE: Try to fence node 'node1'
+info 1280 cmdlist: execute delay 300
+info 1282 node3/crm: got lock 'ha_agent_node1_lock'
+info 1282 node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info 1282 node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai 1282 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info 1282 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info 1282 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4'
+info 1282 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node4)
+info 1285 node4/lrm: got lock 'ha_agent_node4_lock'
+info 1285 node4/lrm: status change wait_for_agent_lock => active
+info 1285 node4/lrm: starting service vm:102
+info 1285 node4/lrm: service status vm:102 started
+info 1660 cmdlist: execute power node1 on
+info 1660 node1/crm: status change startup => wait_for_quorum
+info 1660 node1/lrm: status change startup => wait_for_agent_lock
+info 1660 node1/crm: status change wait_for_quorum => slave
+info 1664 node3/crm: node 'node1': state changed from 'unknown' => 'online'
+info 1664 node3/crm: migrate service 'vm:102' to node 'node1' (running)
+info 1664 node3/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node4, target = node1)
+info 1667 node4/lrm: service vm:102 - start migrate to node 'node1'
+info 1667 node4/lrm: service vm:102 - end migrate to node 'node1'
+info 1684 node3/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1)
+info 1701 node1/lrm: got lock 'ha_agent_node1_lock'
+info 1701 node1/lrm: status change wait_for_agent_lock => active
+info 1701 node1/lrm: starting service vm:102
+info 1701 node1/lrm: service status vm:102 started
+info 1760 cmdlist: execute delay 300
+info 1825 node3/lrm: node had no service configured for 60 rounds, going idle.
+info 1825 node3/lrm: status change active => wait_for_agent_lock
+info 2140 cmdlist: execute power node2 off
+info 2140 node2/crm: killed by poweroff
+info 2140 node2/lrm: killed by poweroff
+info 2142 node3/crm: node 'node2': state changed from 'online' => 'unknown'
+info 2240 cmdlist: execute power node1 off
+info 2240 node1/crm: killed by poweroff
+info 2240 node1/lrm: killed by poweroff
+info 2240 node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info 2300 node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info 2300 node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai 2300 node3/crm: FENCE: Try to fence node 'node1'
+info 2340 cmdlist: execute delay 300
+info 2360 node3/crm: got lock 'ha_agent_node1_lock'
+info 2360 node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info 2360 node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai 2360 node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info 2360 node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info 2360 node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4'
+info 2360 node3/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node4)
+info 2363 node4/lrm: starting service vm:102
+info 2363 node4/lrm: service status vm:102 started
+info 2720 cmdlist: execute power node1 on
+info 2720 node1/crm: status change startup => wait_for_quorum
+info 2720 node1/lrm: status change startup => wait_for_agent_lock
+info 2720 node1/crm: status change wait_for_quorum => slave
+info 2722 node3/crm: node 'node1': state changed from 'unknown' => 'online'
+info 2722 node3/crm: migrate service 'vm:102' to node 'node1' (running)
+info 2722 node3/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node4, target = node1)
+info 2725 node4/lrm: service vm:102 - start migrate to node 'node1'
+info 2725 node4/lrm: service vm:102 - end migrate to node 'node1'
+info 2742 node3/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1)
+info 2761 node1/lrm: got lock 'ha_agent_node1_lock'
+info 2761 node1/lrm: status change wait_for_agent_lock => active
+info 2761 node1/lrm: starting service vm:102
+info 2761 node1/lrm: service status vm:102 started
+info 2820 cmdlist: execute delay 300
+info 3200 cmdlist: execute power node2 on
+info 3200 node2/crm: status change startup => wait_for_quorum
+info 3200 node2/lrm: status change startup => wait_for_agent_lock
+info 3202 node2/crm: status change wait_for_quorum => slave
+info 3204 node3/crm: node 'node2': state changed from 'unknown' => 'online'
+info 3300 cmdlist: execute power node3 off
+info 3300 node3/crm: killed by poweroff
+info 3300 node3/lrm: killed by poweroff
+info 3400 cmdlist: execute power node1 off
+info 3400 node1/crm: killed by poweroff
+info 3400 node1/lrm: killed by poweroff
+info 3420 node2/crm: got lock 'ha_manager_lock'
+info 3420 node2/crm: status change slave => master
+info 3420 node2/crm: using scheduler mode 'static'
+info 3420 node2/crm: node 'node1': state changed from 'online' => 'unknown'
+info 3420 node2/crm: node 'node3': state changed from 'online' => 'unknown'
+info 3480 node2/crm: service 'vm:102': state changed from 'started' to 'fence'
+info 3480 node2/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai 3480 node2/crm: FENCE: Try to fence node 'node1'
+info 3520 node2/crm: got lock 'ha_agent_node1_lock'
+info 3520 node2/crm: fencing: acknowledged - got agent lock for node 'node1'
+info 3520 node2/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai 3520 node2/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info 3520 node2/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info 3520 node2/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4'
+info 3520 node2/crm: service 'vm:102': state changed from 'recovery' to 'started' (node = node4)
+info 3523 node4/lrm: starting service vm:102
+info 3523 node4/lrm: service status vm:102 started
+info 4000 hardware: exit simulation - done
diff --git a/src/test/test-crs-static2/manager_status b/src/test/test-crs-static2/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static2/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static2/service_config b/src/test/test-crs-static2/service_config
new file mode 100644
index 0000000..1f2333d
--- /dev/null
+++ b/src/test/test-crs-static2/service_config
@@ -0,0 +1,3 @@
+{
+ "vm:102": { "node": "node1", "state": "enabled", "group": "prefer_node1" }
+}
diff --git a/src/test/test-crs-static2/static_service_stats b/src/test/test-crs-static2/static_service_stats
new file mode 100644
index 0000000..7fb992d
--- /dev/null
+++ b/src/test/test-crs-static2/static_service_stats
@@ -0,0 +1,3 @@
+{
+ "vm:102": { "maxcpu": 2, "maxmem": 4000000000 }
+}
diff --git a/src/test/test-crs-static3/README b/src/test/test-crs-static3/README
new file mode 100644
index 0000000..db929e1
--- /dev/null
+++ b/src/test/test-crs-static3/README
@@ -0,0 +1,5 @@
+Test how shutdown migrate policy works with the 'static' resource scheduling
+mode.
+
+Expect that when node1 is shut down the services get migrated in the repeating
+sequence node2 node2 node3, because node 2 has twice the resources of node 3.
diff --git a/src/test/test-crs-static3/cmdlist b/src/test/test-crs-static3/cmdlist
new file mode 100644
index 0000000..e84297f
--- /dev/null
+++ b/src/test/test-crs-static3/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "shutdown node1" ]
+]
diff --git a/src/test/test-crs-static3/datacenter.cfg b/src/test/test-crs-static3/datacenter.cfg
new file mode 100644
index 0000000..caa8148
--- /dev/null
+++ b/src/test/test-crs-static3/datacenter.cfg
@@ -0,0 +1,9 @@
+{
+ "crs": {
+ "ha": "static"
+ },
+ "ha": {
+ "shutdown_policy": "migrate"
+ }
+}
+
diff --git a/src/test/test-crs-static3/hardware_status b/src/test/test-crs-static3/hardware_status
new file mode 100644
index 0000000..dfbf496
--- /dev/null
+++ b/src/test/test-crs-static3/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node2": { "power": "off", "network": "off", "cpus": 64, "memory": 200000000000 },
+ "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static3/log.expect b/src/test/test-crs-static3/log.expect
new file mode 100644
index 0000000..00cfefb
--- /dev/null
+++ b/src/test/test-crs-static3/log.expect
@@ -0,0 +1,131 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: using scheduler mode 'static'
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:100' on node 'node1'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node1'
+info 20 node1/crm: adding new service 'vm:103' on node 'node1'
+info 20 node1/crm: adding new service 'vm:104' on node 'node1'
+info 20 node1/crm: adding new service 'vm:105' on node 'node1'
+info 20 node1/crm: adding new service 'vm:106' on node 'node1'
+info 20 node1/crm: adding new service 'vm:107' on node 'node1'
+info 20 node1/crm: adding new service 'vm:108' on node 'node1'
+info 20 node1/crm: adding new service 'vm:109' on node 'node1'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 21 node1/lrm: starting service vm:102
+info 21 node1/lrm: service status vm:102 started
+info 21 node1/lrm: starting service vm:103
+info 21 node1/lrm: service status vm:103 started
+info 21 node1/lrm: starting service vm:104
+info 21 node1/lrm: service status vm:104 started
+info 21 node1/lrm: starting service vm:105
+info 21 node1/lrm: service status vm:105 started
+info 21 node1/lrm: starting service vm:106
+info 21 node1/lrm: service status vm:106 started
+info 21 node1/lrm: starting service vm:107
+info 21 node1/lrm: service status vm:107 started
+info 21 node1/lrm: starting service vm:108
+info 21 node1/lrm: service status vm:108 started
+info 21 node1/lrm: starting service vm:109
+info 21 node1/lrm: service status vm:109 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 24 node3/crm: status change wait_for_quorum => slave
+info 40 node1/crm: service 'vm:100': state changed from 'request_stop' to 'stopped'
+info 120 cmdlist: execute shutdown node1
+info 120 node1/lrm: got shutdown request with shutdown policy 'migrate'
+info 120 node1/lrm: shutdown LRM, doing maintenance, removing this node from active list
+info 120 node1/crm: node 'node1': state changed from 'online' => 'maintenance'
+info 120 node1/crm: migrate service 'vm:101' to node 'node2' (running)
+info 120 node1/crm: service 'vm:101': state changed from 'started' to 'migrate' (node = node1, target = node2)
+info 120 node1/crm: migrate service 'vm:102' to node 'node2' (running)
+info 120 node1/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node1, target = node2)
+info 120 node1/crm: migrate service 'vm:103' to node 'node3' (running)
+info 120 node1/crm: service 'vm:103': state changed from 'started' to 'migrate' (node = node1, target = node3)
+info 120 node1/crm: migrate service 'vm:104' to node 'node2' (running)
+info 120 node1/crm: service 'vm:104': state changed from 'started' to 'migrate' (node = node1, target = node2)
+info 120 node1/crm: migrate service 'vm:105' to node 'node2' (running)
+info 120 node1/crm: service 'vm:105': state changed from 'started' to 'migrate' (node = node1, target = node2)
+info 120 node1/crm: migrate service 'vm:106' to node 'node3' (running)
+info 120 node1/crm: service 'vm:106': state changed from 'started' to 'migrate' (node = node1, target = node3)
+info 120 node1/crm: migrate service 'vm:107' to node 'node2' (running)
+info 120 node1/crm: service 'vm:107': state changed from 'started' to 'migrate' (node = node1, target = node2)
+info 120 node1/crm: migrate service 'vm:108' to node 'node2' (running)
+info 120 node1/crm: service 'vm:108': state changed from 'started' to 'migrate' (node = node1, target = node2)
+info 120 node1/crm: migrate service 'vm:109' to node 'node3' (running)
+info 120 node1/crm: service 'vm:109': state changed from 'started' to 'migrate' (node = node1, target = node3)
+info 121 node1/lrm: status change active => maintenance
+info 121 node1/lrm: service vm:101 - start migrate to node 'node2'
+info 121 node1/lrm: service vm:101 - end migrate to node 'node2'
+info 121 node1/lrm: service vm:102 - start migrate to node 'node2'
+info 121 node1/lrm: service vm:102 - end migrate to node 'node2'
+info 121 node1/lrm: service vm:103 - start migrate to node 'node3'
+info 121 node1/lrm: service vm:103 - end migrate to node 'node3'
+info 121 node1/lrm: service vm:104 - start migrate to node 'node2'
+info 121 node1/lrm: service vm:104 - end migrate to node 'node2'
+info 121 node1/lrm: service vm:105 - start migrate to node 'node2'
+info 121 node1/lrm: service vm:105 - end migrate to node 'node2'
+info 121 node1/lrm: service vm:106 - start migrate to node 'node3'
+info 121 node1/lrm: service vm:106 - end migrate to node 'node3'
+info 121 node1/lrm: service vm:107 - start migrate to node 'node2'
+info 121 node1/lrm: service vm:107 - end migrate to node 'node2'
+info 121 node1/lrm: service vm:108 - start migrate to node 'node2'
+info 121 node1/lrm: service vm:108 - end migrate to node 'node2'
+info 121 node1/lrm: service vm:109 - start migrate to node 'node3'
+info 121 node1/lrm: service vm:109 - end migrate to node 'node3'
+info 140 node1/crm: service 'vm:101': state changed from 'migrate' to 'started' (node = node2)
+info 140 node1/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node2)
+info 140 node1/crm: service 'vm:103': state changed from 'migrate' to 'started' (node = node3)
+info 140 node1/crm: service 'vm:104': state changed from 'migrate' to 'started' (node = node2)
+info 140 node1/crm: service 'vm:105': state changed from 'migrate' to 'started' (node = node2)
+info 140 node1/crm: service 'vm:106': state changed from 'migrate' to 'started' (node = node3)
+info 140 node1/crm: service 'vm:107': state changed from 'migrate' to 'started' (node = node2)
+info 140 node1/crm: service 'vm:108': state changed from 'migrate' to 'started' (node = node2)
+info 140 node1/crm: service 'vm:109': state changed from 'migrate' to 'started' (node = node3)
+info 142 node1/lrm: exit (loop end)
+info 142 shutdown: execute crm node1 stop
+info 141 node1/crm: server received shutdown request
+info 143 node2/lrm: got lock 'ha_agent_node2_lock'
+info 143 node2/lrm: status change wait_for_agent_lock => active
+info 143 node2/lrm: starting service vm:101
+info 143 node2/lrm: service status vm:101 started
+info 143 node2/lrm: starting service vm:102
+info 143 node2/lrm: service status vm:102 started
+info 143 node2/lrm: starting service vm:104
+info 143 node2/lrm: service status vm:104 started
+info 143 node2/lrm: starting service vm:105
+info 143 node2/lrm: service status vm:105 started
+info 143 node2/lrm: starting service vm:107
+info 143 node2/lrm: service status vm:107 started
+info 143 node2/lrm: starting service vm:108
+info 143 node2/lrm: service status vm:108 started
+info 145 node3/lrm: got lock 'ha_agent_node3_lock'
+info 145 node3/lrm: status change wait_for_agent_lock => active
+info 145 node3/lrm: starting service vm:103
+info 145 node3/lrm: service status vm:103 started
+info 145 node3/lrm: starting service vm:106
+info 145 node3/lrm: service status vm:106 started
+info 145 node3/lrm: starting service vm:109
+info 145 node3/lrm: service status vm:109 started
+info 160 node1/crm: voluntary release CRM lock
+info 161 node1/crm: exit (loop end)
+info 161 shutdown: execute power node1 off
+info 161 node2/crm: got lock 'ha_manager_lock'
+info 161 node2/crm: status change slave => master
+info 161 node2/crm: using scheduler mode 'static'
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-crs-static3/manager_status b/src/test/test-crs-static3/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static3/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static3/service_config b/src/test/test-crs-static3/service_config
new file mode 100644
index 0000000..47f94d3
--- /dev/null
+++ b/src/test/test-crs-static3/service_config
@@ -0,0 +1,12 @@
+{
+ "vm:100": { "node": "node1", "state": "stopped" },
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node1", "state": "enabled" },
+ "vm:103": { "node": "node1", "state": "enabled" },
+ "vm:104": { "node": "node1", "state": "enabled" },
+ "vm:105": { "node": "node1", "state": "enabled" },
+ "vm:106": { "node": "node1", "state": "enabled" },
+ "vm:107": { "node": "node1", "state": "enabled" },
+ "vm:108": { "node": "node1", "state": "enabled" },
+ "vm:109": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static3/static_service_stats b/src/test/test-crs-static3/static_service_stats
new file mode 100644
index 0000000..bca71cb
--- /dev/null
+++ b/src/test/test-crs-static3/static_service_stats
@@ -0,0 +1,12 @@
+{
+ "vm:100": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:101": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:102": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:103": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:104": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:105": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:106": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:107": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:108": { "maxcpu": 2, "maxmem": 4000000000 },
+ "vm:109": { "maxcpu": 2, "maxmem": 4000000000 }
+}
diff --git a/src/test/test-crs-static4/README b/src/test/test-crs-static4/README
new file mode 100644
index 0000000..4dfb1bc
--- /dev/null
+++ b/src/test/test-crs-static4/README
@@ -0,0 +1,6 @@
+Test how shutdown migrate policy works with the 'static' resource scheduling
+mode.
+
+Expect that, when node1 is shut down, the first service is migrated to node2 and
+all others to node 3, because the first service is very resource-heavy compared
+to the others.
diff --git a/src/test/test-crs-static4/cmdlist b/src/test/test-crs-static4/cmdlist
new file mode 100644
index 0000000..e84297f
--- /dev/null
+++ b/src/test/test-crs-static4/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "shutdown node1" ]
+]
diff --git a/src/test/test-crs-static4/datacenter.cfg b/src/test/test-crs-static4/datacenter.cfg
new file mode 100644
index 0000000..caa8148
--- /dev/null
+++ b/src/test/test-crs-static4/datacenter.cfg
@@ -0,0 +1,9 @@
+{
+ "crs": {
+ "ha": "static"
+ },
+ "ha": {
+ "shutdown_policy": "migrate"
+ }
+}
+
diff --git a/src/test/test-crs-static4/hardware_status b/src/test/test-crs-static4/hardware_status
new file mode 100644
index 0000000..a83a2dc
--- /dev/null
+++ b/src/test/test-crs-static4/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static4/log.expect b/src/test/test-crs-static4/log.expect
new file mode 100644
index 0000000..3eedc23
--- /dev/null
+++ b/src/test/test-crs-static4/log.expect
@@ -0,0 +1,149 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: using scheduler mode 'static'
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'ct:100' on node 'node1'
+info 20 node1/crm: adding new service 'ct:101' on node 'node1'
+info 20 node1/crm: adding new service 'ct:102' on node 'node1'
+info 20 node1/crm: adding new service 'ct:103' on node 'node1'
+info 20 node1/crm: adding new service 'ct:104' on node 'node1'
+info 20 node1/crm: adding new service 'ct:105' on node 'node1'
+info 20 node1/crm: adding new service 'ct:106' on node 'node1'
+info 20 node1/crm: adding new service 'ct:107' on node 'node1'
+info 20 node1/crm: adding new service 'ct:108' on node 'node1'
+info 20 node1/crm: adding new service 'ct:109' on node 'node1'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service ct:101
+info 21 node1/lrm: service status ct:101 started
+info 21 node1/lrm: starting service ct:102
+info 21 node1/lrm: service status ct:102 started
+info 21 node1/lrm: starting service ct:103
+info 21 node1/lrm: service status ct:103 started
+info 21 node1/lrm: starting service ct:104
+info 21 node1/lrm: service status ct:104 started
+info 21 node1/lrm: starting service ct:105
+info 21 node1/lrm: service status ct:105 started
+info 21 node1/lrm: starting service ct:106
+info 21 node1/lrm: service status ct:106 started
+info 21 node1/lrm: starting service ct:107
+info 21 node1/lrm: service status ct:107 started
+info 21 node1/lrm: starting service ct:108
+info 21 node1/lrm: service status ct:108 started
+info 21 node1/lrm: starting service ct:109
+info 21 node1/lrm: service status ct:109 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 24 node3/crm: status change wait_for_quorum => slave
+info 40 node1/crm: service 'ct:100': state changed from 'request_stop' to 'stopped'
+info 120 cmdlist: execute shutdown node1
+info 120 node1/lrm: got shutdown request with shutdown policy 'migrate'
+info 120 node1/lrm: shutdown LRM, doing maintenance, removing this node from active list
+info 120 node1/crm: node 'node1': state changed from 'online' => 'maintenance'
+info 120 node1/crm: relocate service 'ct:101' to node 'node2'
+info 120 node1/crm: service 'ct:101': state changed from 'started' to 'relocate' (node = node1, target = node2)
+info 120 node1/crm: relocate service 'ct:102' to node 'node3'
+info 120 node1/crm: service 'ct:102': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:103' to node 'node3'
+info 120 node1/crm: service 'ct:103': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:104' to node 'node3'
+info 120 node1/crm: service 'ct:104': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:105' to node 'node3'
+info 120 node1/crm: service 'ct:105': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:106' to node 'node3'
+info 120 node1/crm: service 'ct:106': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:107' to node 'node3'
+info 120 node1/crm: service 'ct:107': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:108' to node 'node3'
+info 120 node1/crm: service 'ct:108': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:109' to node 'node3'
+info 120 node1/crm: service 'ct:109': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 121 node1/lrm: status change active => maintenance
+info 121 node1/lrm: service ct:101 - start relocate to node 'node2'
+info 121 node1/lrm: stopping service ct:101 (relocate)
+info 121 node1/lrm: service status ct:101 stopped
+info 121 node1/lrm: service ct:101 - end relocate to node 'node2'
+info 121 node1/lrm: service ct:102 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:102 (relocate)
+info 121 node1/lrm: service status ct:102 stopped
+info 121 node1/lrm: service ct:102 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:103 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:103 (relocate)
+info 121 node1/lrm: service status ct:103 stopped
+info 121 node1/lrm: service ct:103 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:104 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:104 (relocate)
+info 121 node1/lrm: service status ct:104 stopped
+info 121 node1/lrm: service ct:104 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:105 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:105 (relocate)
+info 121 node1/lrm: service status ct:105 stopped
+info 121 node1/lrm: service ct:105 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:106 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:106 (relocate)
+info 121 node1/lrm: service status ct:106 stopped
+info 121 node1/lrm: service ct:106 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:107 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:107 (relocate)
+info 121 node1/lrm: service status ct:107 stopped
+info 121 node1/lrm: service ct:107 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:108 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:108 (relocate)
+info 121 node1/lrm: service status ct:108 stopped
+info 121 node1/lrm: service ct:108 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:109 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:109 (relocate)
+info 121 node1/lrm: service status ct:109 stopped
+info 121 node1/lrm: service ct:109 - end relocate to node 'node3'
+info 140 node1/crm: service 'ct:101': state changed from 'relocate' to 'started' (node = node2)
+info 140 node1/crm: service 'ct:102': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:103': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:104': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:105': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:106': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:107': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:108': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:109': state changed from 'relocate' to 'started' (node = node3)
+info 142 node1/lrm: exit (loop end)
+info 142 shutdown: execute crm node1 stop
+info 141 node1/crm: server received shutdown request
+info 143 node2/lrm: got lock 'ha_agent_node2_lock'
+info 143 node2/lrm: status change wait_for_agent_lock => active
+info 143 node2/lrm: starting service ct:101
+info 143 node2/lrm: service status ct:101 started
+info 145 node3/lrm: got lock 'ha_agent_node3_lock'
+info 145 node3/lrm: status change wait_for_agent_lock => active
+info 145 node3/lrm: starting service ct:102
+info 145 node3/lrm: service status ct:102 started
+info 145 node3/lrm: starting service ct:103
+info 145 node3/lrm: service status ct:103 started
+info 145 node3/lrm: starting service ct:104
+info 145 node3/lrm: service status ct:104 started
+info 145 node3/lrm: starting service ct:105
+info 145 node3/lrm: service status ct:105 started
+info 145 node3/lrm: starting service ct:106
+info 145 node3/lrm: service status ct:106 started
+info 145 node3/lrm: starting service ct:107
+info 145 node3/lrm: service status ct:107 started
+info 145 node3/lrm: starting service ct:108
+info 145 node3/lrm: service status ct:108 started
+info 145 node3/lrm: starting service ct:109
+info 145 node3/lrm: service status ct:109 started
+info 160 node1/crm: voluntary release CRM lock
+info 161 node1/crm: exit (loop end)
+info 161 shutdown: execute power node1 off
+info 161 node2/crm: got lock 'ha_manager_lock'
+info 161 node2/crm: status change slave => master
+info 161 node2/crm: using scheduler mode 'static'
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-crs-static4/manager_status b/src/test/test-crs-static4/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static4/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static4/service_config b/src/test/test-crs-static4/service_config
new file mode 100644
index 0000000..b984a09
--- /dev/null
+++ b/src/test/test-crs-static4/service_config
@@ -0,0 +1,12 @@
+{
+ "ct:100": { "node": "node1", "state": "stopped" },
+ "ct:101": { "node": "node1", "state": "enabled" },
+ "ct:102": { "node": "node1", "state": "enabled" },
+ "ct:103": { "node": "node1", "state": "enabled" },
+ "ct:104": { "node": "node1", "state": "enabled" },
+ "ct:105": { "node": "node1", "state": "enabled" },
+ "ct:106": { "node": "node1", "state": "enabled" },
+ "ct:107": { "node": "node1", "state": "enabled" },
+ "ct:108": { "node": "node1", "state": "enabled" },
+ "ct:109": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static4/static_service_stats b/src/test/test-crs-static4/static_service_stats
new file mode 100644
index 0000000..878709b
--- /dev/null
+++ b/src/test/test-crs-static4/static_service_stats
@@ -0,0 +1,12 @@
+{
+ "ct:100": { "maxcpu": 2, "maxmem": 4000000000 },
+ "ct:101": { "maxcpu": 0, "maxmem": 40000000000 },
+ "ct:102": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:103": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:104": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:105": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:106": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:107": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:108": { "maxcpu": 2, "maxmem": 2000000000 },
+ "ct:109": { "maxcpu": 2, "maxmem": 2000000000 }
+}
diff --git a/src/test/test-crs-static5/README b/src/test/test-crs-static5/README
new file mode 100644
index 0000000..d9b5dc7
--- /dev/null
+++ b/src/test/test-crs-static5/README
@@ -0,0 +1,5 @@
+Test how recovery works with the 'static' resource scheduling mode.
+
+Expect that, when node1 is shut down, all services are migrated to node 3,
+because the services don't have much memory, node 2 and 3 both already have a
+service with high memory, but node 3 has much left-over CPU.
diff --git a/src/test/test-crs-static5/cmdlist b/src/test/test-crs-static5/cmdlist
new file mode 100644
index 0000000..e84297f
--- /dev/null
+++ b/src/test/test-crs-static5/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "shutdown node1" ]
+]
diff --git a/src/test/test-crs-static5/datacenter.cfg b/src/test/test-crs-static5/datacenter.cfg
new file mode 100644
index 0000000..caa8148
--- /dev/null
+++ b/src/test/test-crs-static5/datacenter.cfg
@@ -0,0 +1,9 @@
+{
+ "crs": {
+ "ha": "static"
+ },
+ "ha": {
+ "shutdown_policy": "migrate"
+ }
+}
+
diff --git a/src/test/test-crs-static5/hardware_status b/src/test/test-crs-static5/hardware_status
new file mode 100644
index 0000000..3eb9e73
--- /dev/null
+++ b/src/test/test-crs-static5/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+ "node3": { "power": "off", "network": "off", "cpus": 128, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static5/log.expect b/src/test/test-crs-static5/log.expect
new file mode 100644
index 0000000..cb6b0d5
--- /dev/null
+++ b/src/test/test-crs-static5/log.expect
@@ -0,0 +1,117 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: using scheduler mode 'static'
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'ct:102' on node 'node2'
+info 20 node1/crm: adding new service 'ct:103' on node 'node3'
+info 20 node1/crm: adding new service 'ct:104' on node 'node1'
+info 20 node1/crm: adding new service 'ct:105' on node 'node1'
+info 20 node1/crm: adding new service 'ct:106' on node 'node1'
+info 20 node1/crm: adding new service 'ct:107' on node 'node1'
+info 20 node1/crm: adding new service 'ct:108' on node 'node1'
+info 20 node1/crm: adding new service 'ct:109' on node 'node1'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service ct:104
+info 21 node1/lrm: service status ct:104 started
+info 21 node1/lrm: starting service ct:105
+info 21 node1/lrm: service status ct:105 started
+info 21 node1/lrm: starting service ct:106
+info 21 node1/lrm: service status ct:106 started
+info 21 node1/lrm: starting service ct:107
+info 21 node1/lrm: service status ct:107 started
+info 21 node1/lrm: starting service ct:108
+info 21 node1/lrm: service status ct:108 started
+info 21 node1/lrm: starting service ct:109
+info 21 node1/lrm: service status ct:109 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service ct:102
+info 23 node2/lrm: service status ct:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service ct:103
+info 25 node3/lrm: service status ct:103 started
+info 120 cmdlist: execute shutdown node1
+info 120 node1/lrm: got shutdown request with shutdown policy 'migrate'
+info 120 node1/lrm: shutdown LRM, doing maintenance, removing this node from active list
+info 120 node1/crm: node 'node1': state changed from 'online' => 'maintenance'
+info 120 node1/crm: relocate service 'ct:104' to node 'node3'
+info 120 node1/crm: service 'ct:104': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:105' to node 'node3'
+info 120 node1/crm: service 'ct:105': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:106' to node 'node3'
+info 120 node1/crm: service 'ct:106': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:107' to node 'node3'
+info 120 node1/crm: service 'ct:107': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:108' to node 'node3'
+info 120 node1/crm: service 'ct:108': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 120 node1/crm: relocate service 'ct:109' to node 'node3'
+info 120 node1/crm: service 'ct:109': state changed from 'started' to 'relocate' (node = node1, target = node3)
+info 121 node1/lrm: status change active => maintenance
+info 121 node1/lrm: service ct:104 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:104 (relocate)
+info 121 node1/lrm: service status ct:104 stopped
+info 121 node1/lrm: service ct:104 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:105 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:105 (relocate)
+info 121 node1/lrm: service status ct:105 stopped
+info 121 node1/lrm: service ct:105 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:106 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:106 (relocate)
+info 121 node1/lrm: service status ct:106 stopped
+info 121 node1/lrm: service ct:106 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:107 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:107 (relocate)
+info 121 node1/lrm: service status ct:107 stopped
+info 121 node1/lrm: service ct:107 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:108 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:108 (relocate)
+info 121 node1/lrm: service status ct:108 stopped
+info 121 node1/lrm: service ct:108 - end relocate to node 'node3'
+info 121 node1/lrm: service ct:109 - start relocate to node 'node3'
+info 121 node1/lrm: stopping service ct:109 (relocate)
+info 121 node1/lrm: service status ct:109 stopped
+info 121 node1/lrm: service ct:109 - end relocate to node 'node3'
+info 140 node1/crm: service 'ct:104': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:105': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:106': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:107': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:108': state changed from 'relocate' to 'started' (node = node3)
+info 140 node1/crm: service 'ct:109': state changed from 'relocate' to 'started' (node = node3)
+info 142 node1/lrm: exit (loop end)
+info 142 shutdown: execute crm node1 stop
+info 141 node1/crm: server received shutdown request
+info 145 node3/lrm: starting service ct:104
+info 145 node3/lrm: service status ct:104 started
+info 145 node3/lrm: starting service ct:105
+info 145 node3/lrm: service status ct:105 started
+info 145 node3/lrm: starting service ct:106
+info 145 node3/lrm: service status ct:106 started
+info 145 node3/lrm: starting service ct:107
+info 145 node3/lrm: service status ct:107 started
+info 145 node3/lrm: starting service ct:108
+info 145 node3/lrm: service status ct:108 started
+info 145 node3/lrm: starting service ct:109
+info 145 node3/lrm: service status ct:109 started
+info 160 node1/crm: voluntary release CRM lock
+info 161 node1/crm: exit (loop end)
+info 161 shutdown: execute power node1 off
+info 161 node2/crm: got lock 'ha_manager_lock'
+info 161 node2/crm: status change slave => master
+info 161 node2/crm: using scheduler mode 'static'
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-crs-static5/manager_status b/src/test/test-crs-static5/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static5/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static5/service_config b/src/test/test-crs-static5/service_config
new file mode 100644
index 0000000..43c5f60
--- /dev/null
+++ b/src/test/test-crs-static5/service_config
@@ -0,0 +1,10 @@
+{
+ "ct:102": { "node": "node2", "state": "enabled" },
+ "ct:103": { "node": "node3", "state": "enabled" },
+ "ct:104": { "node": "node1", "state": "enabled" },
+ "ct:105": { "node": "node1", "state": "enabled" },
+ "ct:106": { "node": "node1", "state": "enabled" },
+ "ct:107": { "node": "node1", "state": "enabled" },
+ "ct:108": { "node": "node1", "state": "enabled" },
+ "ct:109": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static5/static_service_stats b/src/test/test-crs-static5/static_service_stats
new file mode 100644
index 0000000..6293f63
--- /dev/null
+++ b/src/test/test-crs-static5/static_service_stats
@@ -0,0 +1,11 @@
+{
+ "ct:101": { "maxcpu": 0, "maxmem": 40000000000 },
+ "ct:102": { "maxcpu": 0.5, "maxmem": 40000000000 },
+ "ct:103": { "maxcpu": 0.5, "maxmem": 200000000 },
+ "ct:104": { "maxcpu": 0.5, "maxmem": 200000000 },
+ "ct:105": { "maxcpu": 0.5, "maxmem": 200000000 },
+ "ct:106": { "maxcpu": 0.5, "maxmem": 200000000 },
+ "ct:107": { "maxcpu": 0.5, "maxmem": 200000000 },
+ "ct:108": { "maxcpu": 0.5, "maxmem": 200000000 },
+ "ct:109": { "maxcpu": 0.5, "maxmem": 200000000 }
+}
--
2.30.2
More information about the pve-devel
mailing list