[pve-devel] [PATCH v2 ha-manager 14/15] test: add tests for static resource scheduling

Fiona Ebner f.ebner at proxmox.com
Thu Nov 17 15:00:15 CET 2022


See the READMEs for more information about the tests.

Signed-off-by: Fiona Ebner <f.ebner at proxmox.com>
---

New in v2.

 src/test/test-crs-static1/README              |   4 +
 src/test/test-crs-static1/cmdlist             |   4 +
 src/test/test-crs-static1/datacenter.cfg      |   6 +
 src/test/test-crs-static1/hardware_status     |   5 +
 src/test/test-crs-static1/log.expect          |  50 +++++
 src/test/test-crs-static1/manager_status      |   1 +
 src/test/test-crs-static1/service_config      |   3 +
 .../test-crs-static1/static_service_stats     |   3 +
 src/test/test-crs-static2/README              |   4 +
 src/test/test-crs-static2/cmdlist             |  20 ++
 src/test/test-crs-static2/datacenter.cfg      |   6 +
 src/test/test-crs-static2/groups              |   2 +
 src/test/test-crs-static2/hardware_status     |   7 +
 src/test/test-crs-static2/log.expect          | 171 ++++++++++++++++++
 src/test/test-crs-static2/manager_status      |   1 +
 src/test/test-crs-static2/service_config      |   3 +
 .../test-crs-static2/static_service_stats     |   3 +
 src/test/test-crs-static3/README              |   5 +
 src/test/test-crs-static3/cmdlist             |   4 +
 src/test/test-crs-static3/datacenter.cfg      |   9 +
 src/test/test-crs-static3/hardware_status     |   5 +
 src/test/test-crs-static3/log.expect          | 131 ++++++++++++++
 src/test/test-crs-static3/manager_status      |   1 +
 src/test/test-crs-static3/service_config      |  12 ++
 .../test-crs-static3/static_service_stats     |  12 ++
 src/test/test-crs-static4/README              |   6 +
 src/test/test-crs-static4/cmdlist             |   4 +
 src/test/test-crs-static4/datacenter.cfg      |   9 +
 src/test/test-crs-static4/hardware_status     |   5 +
 src/test/test-crs-static4/log.expect          | 149 +++++++++++++++
 src/test/test-crs-static4/manager_status      |   1 +
 src/test/test-crs-static4/service_config      |  12 ++
 .../test-crs-static4/static_service_stats     |  12 ++
 src/test/test-crs-static5/README              |   5 +
 src/test/test-crs-static5/cmdlist             |   4 +
 src/test/test-crs-static5/datacenter.cfg      |   9 +
 src/test/test-crs-static5/hardware_status     |   5 +
 src/test/test-crs-static5/log.expect          | 117 ++++++++++++
 src/test/test-crs-static5/manager_status      |   1 +
 src/test/test-crs-static5/service_config      |  10 +
 .../test-crs-static5/static_service_stats     |  11 ++
 41 files changed, 832 insertions(+)
 create mode 100644 src/test/test-crs-static1/README
 create mode 100644 src/test/test-crs-static1/cmdlist
 create mode 100644 src/test/test-crs-static1/datacenter.cfg
 create mode 100644 src/test/test-crs-static1/hardware_status
 create mode 100644 src/test/test-crs-static1/log.expect
 create mode 100644 src/test/test-crs-static1/manager_status
 create mode 100644 src/test/test-crs-static1/service_config
 create mode 100644 src/test/test-crs-static1/static_service_stats
 create mode 100644 src/test/test-crs-static2/README
 create mode 100644 src/test/test-crs-static2/cmdlist
 create mode 100644 src/test/test-crs-static2/datacenter.cfg
 create mode 100644 src/test/test-crs-static2/groups
 create mode 100644 src/test/test-crs-static2/hardware_status
 create mode 100644 src/test/test-crs-static2/log.expect
 create mode 100644 src/test/test-crs-static2/manager_status
 create mode 100644 src/test/test-crs-static2/service_config
 create mode 100644 src/test/test-crs-static2/static_service_stats
 create mode 100644 src/test/test-crs-static3/README
 create mode 100644 src/test/test-crs-static3/cmdlist
 create mode 100644 src/test/test-crs-static3/datacenter.cfg
 create mode 100644 src/test/test-crs-static3/hardware_status
 create mode 100644 src/test/test-crs-static3/log.expect
 create mode 100644 src/test/test-crs-static3/manager_status
 create mode 100644 src/test/test-crs-static3/service_config
 create mode 100644 src/test/test-crs-static3/static_service_stats
 create mode 100644 src/test/test-crs-static4/README
 create mode 100644 src/test/test-crs-static4/cmdlist
 create mode 100644 src/test/test-crs-static4/datacenter.cfg
 create mode 100644 src/test/test-crs-static4/hardware_status
 create mode 100644 src/test/test-crs-static4/log.expect
 create mode 100644 src/test/test-crs-static4/manager_status
 create mode 100644 src/test/test-crs-static4/service_config
 create mode 100644 src/test/test-crs-static4/static_service_stats
 create mode 100644 src/test/test-crs-static5/README
 create mode 100644 src/test/test-crs-static5/cmdlist
 create mode 100644 src/test/test-crs-static5/datacenter.cfg
 create mode 100644 src/test/test-crs-static5/hardware_status
 create mode 100644 src/test/test-crs-static5/log.expect
 create mode 100644 src/test/test-crs-static5/manager_status
 create mode 100644 src/test/test-crs-static5/service_config
 create mode 100644 src/test/test-crs-static5/static_service_stats

diff --git a/src/test/test-crs-static1/README b/src/test/test-crs-static1/README
new file mode 100644
index 0000000..483f265
--- /dev/null
+++ b/src/test/test-crs-static1/README
@@ -0,0 +1,4 @@
+Test how service recovery works with the 'static' resource scheduling mode.
+
+Expect that the single service gets recovered to the node with the most
+available resources.
diff --git a/src/test/test-crs-static1/cmdlist b/src/test/test-crs-static1/cmdlist
new file mode 100644
index 0000000..8684073
--- /dev/null
+++ b/src/test/test-crs-static1/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "network node1 off" ]
+]
diff --git a/src/test/test-crs-static1/datacenter.cfg b/src/test/test-crs-static1/datacenter.cfg
new file mode 100644
index 0000000..8f83457
--- /dev/null
+++ b/src/test/test-crs-static1/datacenter.cfg
@@ -0,0 +1,6 @@
+{
+    "crs": {
+        "ha": "static"
+    }
+}
+
diff --git a/src/test/test-crs-static1/hardware_status b/src/test/test-crs-static1/hardware_status
new file mode 100644
index 0000000..0fa8c26
--- /dev/null
+++ b/src/test/test-crs-static1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 200000000000 },
+  "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 300000000000 }
+}
diff --git a/src/test/test-crs-static1/log.expect b/src/test/test-crs-static1/log.expect
new file mode 100644
index 0000000..2b06b3c
--- /dev/null
+++ b/src/test/test-crs-static1/log.expect
@@ -0,0 +1,50 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: using scheduler mode 'static'
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:102' on node 'node1'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:102
+info     21    node1/lrm: service status vm:102 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info    120      cmdlist: execute network node1 off
+info    120    node1/crm: status change master => lost_manager_lock
+info    120    node1/crm: status change lost_manager_lock => wait_for_quorum
+info    121    node1/lrm: status change active => lost_agent_lock
+info    162     watchdog: execute power node1 off
+info    161    node1/crm: killed by poweroff
+info    162    node1/lrm: killed by poweroff
+info    162     hardware: server 'node1' stopped by poweroff (watchdog)
+info    222    node3/crm: got lock 'ha_manager_lock'
+info    222    node3/crm: status change slave => master
+info    222    node3/crm: using scheduler mode 'static'
+info    222    node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info    282    node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info    282    node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai    282    node3/crm: FENCE: Try to fence node 'node1'
+info    282    node3/crm: got lock 'ha_agent_node1_lock'
+info    282    node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info    282    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai    282    node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info    282    node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info    282    node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node3'
+info    282    node3/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node3)
+info    283    node3/lrm: got lock 'ha_agent_node3_lock'
+info    283    node3/lrm: status change wait_for_agent_lock => active
+info    283    node3/lrm: starting service vm:102
+info    283    node3/lrm: service status vm:102 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-crs-static1/manager_status b/src/test/test-crs-static1/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static1/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static1/service_config b/src/test/test-crs-static1/service_config
new file mode 100644
index 0000000..9c12447
--- /dev/null
+++ b/src/test/test-crs-static1/service_config
@@ -0,0 +1,3 @@
+{
+    "vm:102": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static1/static_service_stats b/src/test/test-crs-static1/static_service_stats
new file mode 100644
index 0000000..7fb992d
--- /dev/null
+++ b/src/test/test-crs-static1/static_service_stats
@@ -0,0 +1,3 @@
+{
+    "vm:102": { "maxcpu": 2, "maxmem": 4000000000 }
+}
diff --git a/src/test/test-crs-static2/README b/src/test/test-crs-static2/README
new file mode 100644
index 0000000..61530a7
--- /dev/null
+++ b/src/test/test-crs-static2/README
@@ -0,0 +1,4 @@
+Test how service recovery works with the 'static' resource scheduling mode.
+
+Expect that the single service always gets recovered to the node with the most
+available resources. Also tests that the group priority still takes precedence.
diff --git a/src/test/test-crs-static2/cmdlist b/src/test/test-crs-static2/cmdlist
new file mode 100644
index 0000000..bada1bb
--- /dev/null
+++ b/src/test/test-crs-static2/cmdlist
@@ -0,0 +1,20 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on", "power node5 on" ],
+    [ "power node1 off" ],
+    [ "delay 300" ],
+    [ "power node1 on" ],
+    [ "delay 300" ],
+    [ "power node4 on" ],
+    [ "power node1 off" ],
+    [ "delay 300" ],
+    [ "power node1 on" ],
+    [ "delay 300" ],
+    [ "power node2 off" ],
+    [ "power node1 off" ],
+    [ "delay 300" ],
+    [ "power node1 on" ],
+    [ "delay 300" ],
+    [ "power node2 on" ],
+    [ "power node3 off" ],
+    [ "power node1 off" ]
+]
diff --git a/src/test/test-crs-static2/datacenter.cfg b/src/test/test-crs-static2/datacenter.cfg
new file mode 100644
index 0000000..8f83457
--- /dev/null
+++ b/src/test/test-crs-static2/datacenter.cfg
@@ -0,0 +1,6 @@
+{
+    "crs": {
+        "ha": "static"
+    }
+}
+
diff --git a/src/test/test-crs-static2/groups b/src/test/test-crs-static2/groups
new file mode 100644
index 0000000..43e9bf5
--- /dev/null
+++ b/src/test/test-crs-static2/groups
@@ -0,0 +1,2 @@
+group: prefer_node1
+        nodes node1
diff --git a/src/test/test-crs-static2/hardware_status b/src/test/test-crs-static2/hardware_status
new file mode 100644
index 0000000..d426023
--- /dev/null
+++ b/src/test/test-crs-static2/hardware_status
@@ -0,0 +1,7 @@
+{
+  "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 200000000000 },
+  "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 300000000000 },
+  "node4": { "power": "off", "network": "off", "cpus": 64, "memory": 300000000000 },
+  "node5": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static2/log.expect b/src/test/test-crs-static2/log.expect
new file mode 100644
index 0000000..ee4416c
--- /dev/null
+++ b/src/test/test-crs-static2/log.expect
@@ -0,0 +1,171 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node5 on
+info     20    node5/crm: status change startup => wait_for_quorum
+info     20    node5/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: using scheduler mode 'static'
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node5': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:102' on node 'node1'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:102
+info     21    node1/lrm: service status vm:102 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     26    node5/crm: status change wait_for_quorum => slave
+info    120      cmdlist: execute power node1 off
+info    120    node1/crm: killed by poweroff
+info    120    node1/lrm: killed by poweroff
+info    220      cmdlist: execute delay 300
+info    222    node3/crm: got lock 'ha_manager_lock'
+info    222    node3/crm: status change slave => master
+info    222    node3/crm: using scheduler mode 'static'
+info    222    node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info    282    node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info    282    node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai    282    node3/crm: FENCE: Try to fence node 'node1'
+info    282    node3/crm: got lock 'ha_agent_node1_lock'
+info    282    node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info    282    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai    282    node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info    282    node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info    282    node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node3'
+info    282    node3/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node3)
+info    283    node3/lrm: got lock 'ha_agent_node3_lock'
+info    283    node3/lrm: status change wait_for_agent_lock => active
+info    283    node3/lrm: starting service vm:102
+info    283    node3/lrm: service status vm:102 started
+info    600      cmdlist: execute power node1 on
+info    600    node1/crm: status change startup => wait_for_quorum
+info    600    node1/lrm: status change startup => wait_for_agent_lock
+info    600    node1/crm: status change wait_for_quorum => slave
+info    604    node3/crm: node 'node1': state changed from 'unknown' => 'online'
+info    604    node3/crm: migrate service 'vm:102' to node 'node1' (running)
+info    604    node3/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    605    node3/lrm: service vm:102 - start migrate to node 'node1'
+info    605    node3/lrm: service vm:102 - end migrate to node 'node1'
+info    624    node3/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node1)
+info    641    node1/lrm: got lock 'ha_agent_node1_lock'
+info    641    node1/lrm: status change wait_for_agent_lock => active
+info    641    node1/lrm: starting service vm:102
+info    641    node1/lrm: service status vm:102 started
+info    700      cmdlist: execute delay 300
+info   1080      cmdlist: execute power node4 on
+info   1080    node4/crm: status change startup => wait_for_quorum
+info   1080    node4/lrm: status change startup => wait_for_agent_lock
+info   1084    node3/crm: node 'node4': state changed from 'unknown' => 'online'
+info   1086    node4/crm: status change wait_for_quorum => slave
+info   1180      cmdlist: execute power node1 off
+info   1180    node1/crm: killed by poweroff
+info   1180    node1/lrm: killed by poweroff
+info   1182    node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info   1242    node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info   1242    node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai   1242    node3/crm: FENCE: Try to fence node 'node1'
+info   1280      cmdlist: execute delay 300
+info   1282    node3/crm: got lock 'ha_agent_node1_lock'
+info   1282    node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info   1282    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai   1282    node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info   1282    node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info   1282    node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4'
+info   1282    node3/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node4)
+info   1285    node4/lrm: got lock 'ha_agent_node4_lock'
+info   1285    node4/lrm: status change wait_for_agent_lock => active
+info   1285    node4/lrm: starting service vm:102
+info   1285    node4/lrm: service status vm:102 started
+info   1660      cmdlist: execute power node1 on
+info   1660    node1/crm: status change startup => wait_for_quorum
+info   1660    node1/lrm: status change startup => wait_for_agent_lock
+info   1660    node1/crm: status change wait_for_quorum => slave
+info   1664    node3/crm: node 'node1': state changed from 'unknown' => 'online'
+info   1664    node3/crm: migrate service 'vm:102' to node 'node1' (running)
+info   1664    node3/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node4, target = node1)
+info   1667    node4/lrm: service vm:102 - start migrate to node 'node1'
+info   1667    node4/lrm: service vm:102 - end migrate to node 'node1'
+info   1684    node3/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node1)
+info   1701    node1/lrm: got lock 'ha_agent_node1_lock'
+info   1701    node1/lrm: status change wait_for_agent_lock => active
+info   1701    node1/lrm: starting service vm:102
+info   1701    node1/lrm: service status vm:102 started
+info   1760      cmdlist: execute delay 300
+info   1825    node3/lrm: node had no service configured for 60 rounds, going idle.
+info   1825    node3/lrm: status change active => wait_for_agent_lock
+info   2140      cmdlist: execute power node2 off
+info   2140    node2/crm: killed by poweroff
+info   2140    node2/lrm: killed by poweroff
+info   2142    node3/crm: node 'node2': state changed from 'online' => 'unknown'
+info   2240      cmdlist: execute power node1 off
+info   2240    node1/crm: killed by poweroff
+info   2240    node1/lrm: killed by poweroff
+info   2240    node3/crm: node 'node1': state changed from 'online' => 'unknown'
+info   2300    node3/crm: service 'vm:102': state changed from 'started' to 'fence'
+info   2300    node3/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai   2300    node3/crm: FENCE: Try to fence node 'node1'
+info   2340      cmdlist: execute delay 300
+info   2360    node3/crm: got lock 'ha_agent_node1_lock'
+info   2360    node3/crm: fencing: acknowledged - got agent lock for node 'node1'
+info   2360    node3/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai   2360    node3/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info   2360    node3/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info   2360    node3/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4'
+info   2360    node3/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node4)
+info   2363    node4/lrm: starting service vm:102
+info   2363    node4/lrm: service status vm:102 started
+info   2720      cmdlist: execute power node1 on
+info   2720    node1/crm: status change startup => wait_for_quorum
+info   2720    node1/lrm: status change startup => wait_for_agent_lock
+info   2720    node1/crm: status change wait_for_quorum => slave
+info   2722    node3/crm: node 'node1': state changed from 'unknown' => 'online'
+info   2722    node3/crm: migrate service 'vm:102' to node 'node1' (running)
+info   2722    node3/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node4, target = node1)
+info   2725    node4/lrm: service vm:102 - start migrate to node 'node1'
+info   2725    node4/lrm: service vm:102 - end migrate to node 'node1'
+info   2742    node3/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node1)
+info   2761    node1/lrm: got lock 'ha_agent_node1_lock'
+info   2761    node1/lrm: status change wait_for_agent_lock => active
+info   2761    node1/lrm: starting service vm:102
+info   2761    node1/lrm: service status vm:102 started
+info   2820      cmdlist: execute delay 300
+info   3200      cmdlist: execute power node2 on
+info   3200    node2/crm: status change startup => wait_for_quorum
+info   3200    node2/lrm: status change startup => wait_for_agent_lock
+info   3202    node2/crm: status change wait_for_quorum => slave
+info   3204    node3/crm: node 'node2': state changed from 'unknown' => 'online'
+info   3300      cmdlist: execute power node3 off
+info   3300    node3/crm: killed by poweroff
+info   3300    node3/lrm: killed by poweroff
+info   3400      cmdlist: execute power node1 off
+info   3400    node1/crm: killed by poweroff
+info   3400    node1/lrm: killed by poweroff
+info   3420    node2/crm: got lock 'ha_manager_lock'
+info   3420    node2/crm: status change slave => master
+info   3420    node2/crm: using scheduler mode 'static'
+info   3420    node2/crm: node 'node1': state changed from 'online' => 'unknown'
+info   3420    node2/crm: node 'node3': state changed from 'online' => 'unknown'
+info   3480    node2/crm: service 'vm:102': state changed from 'started' to 'fence'
+info   3480    node2/crm: node 'node1': state changed from 'unknown' => 'fence'
+emai   3480    node2/crm: FENCE: Try to fence node 'node1'
+info   3520    node2/crm: got lock 'ha_agent_node1_lock'
+info   3520    node2/crm: fencing: acknowledged - got agent lock for node 'node1'
+info   3520    node2/crm: node 'node1': state changed from 'fence' => 'unknown'
+emai   3520    node2/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node1'
+info   3520    node2/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info   3520    node2/crm: recover service 'vm:102' from fenced node 'node1' to node 'node4'
+info   3520    node2/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node4)
+info   3523    node4/lrm: starting service vm:102
+info   3523    node4/lrm: service status vm:102 started
+info   4000     hardware: exit simulation - done
diff --git a/src/test/test-crs-static2/manager_status b/src/test/test-crs-static2/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static2/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static2/service_config b/src/test/test-crs-static2/service_config
new file mode 100644
index 0000000..1f2333d
--- /dev/null
+++ b/src/test/test-crs-static2/service_config
@@ -0,0 +1,3 @@
+{
+    "vm:102": { "node": "node1", "state": "enabled", "group": "prefer_node1" }
+}
diff --git a/src/test/test-crs-static2/static_service_stats b/src/test/test-crs-static2/static_service_stats
new file mode 100644
index 0000000..7fb992d
--- /dev/null
+++ b/src/test/test-crs-static2/static_service_stats
@@ -0,0 +1,3 @@
+{
+    "vm:102": { "maxcpu": 2, "maxmem": 4000000000 }
+}
diff --git a/src/test/test-crs-static3/README b/src/test/test-crs-static3/README
new file mode 100644
index 0000000..db929e1
--- /dev/null
+++ b/src/test/test-crs-static3/README
@@ -0,0 +1,5 @@
+Test how shutdown migrate policy works with the 'static' resource scheduling
+mode.
+
+Expect that when node1 is shut down the services get migrated in the repeating
+sequence node2 node2 node3, because node 2 has twice the resources of node 3.
diff --git a/src/test/test-crs-static3/cmdlist b/src/test/test-crs-static3/cmdlist
new file mode 100644
index 0000000..e84297f
--- /dev/null
+++ b/src/test/test-crs-static3/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "shutdown node1" ]
+]
diff --git a/src/test/test-crs-static3/datacenter.cfg b/src/test/test-crs-static3/datacenter.cfg
new file mode 100644
index 0000000..caa8148
--- /dev/null
+++ b/src/test/test-crs-static3/datacenter.cfg
@@ -0,0 +1,9 @@
+{
+    "crs": {
+        "ha": "static"
+    },
+    "ha": {
+        "shutdown_policy": "migrate"
+    }
+}
+
diff --git a/src/test/test-crs-static3/hardware_status b/src/test/test-crs-static3/hardware_status
new file mode 100644
index 0000000..dfbf496
--- /dev/null
+++ b/src/test/test-crs-static3/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node2": { "power": "off", "network": "off", "cpus": 64, "memory": 200000000000 },
+  "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static3/log.expect b/src/test/test-crs-static3/log.expect
new file mode 100644
index 0000000..00cfefb
--- /dev/null
+++ b/src/test/test-crs-static3/log.expect
@@ -0,0 +1,131 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: using scheduler mode 'static'
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:100' on node 'node1'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node1'
+info     20    node1/crm: adding new service 'vm:103' on node 'node1'
+info     20    node1/crm: adding new service 'vm:104' on node 'node1'
+info     20    node1/crm: adding new service 'vm:105' on node 'node1'
+info     20    node1/crm: adding new service 'vm:106' on node 'node1'
+info     20    node1/crm: adding new service 'vm:107' on node 'node1'
+info     20    node1/crm: adding new service 'vm:108' on node 'node1'
+info     20    node1/crm: adding new service 'vm:109' on node 'node1'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     21    node1/lrm: starting service vm:102
+info     21    node1/lrm: service status vm:102 started
+info     21    node1/lrm: starting service vm:103
+info     21    node1/lrm: service status vm:103 started
+info     21    node1/lrm: starting service vm:104
+info     21    node1/lrm: service status vm:104 started
+info     21    node1/lrm: starting service vm:105
+info     21    node1/lrm: service status vm:105 started
+info     21    node1/lrm: starting service vm:106
+info     21    node1/lrm: service status vm:106 started
+info     21    node1/lrm: starting service vm:107
+info     21    node1/lrm: service status vm:107 started
+info     21    node1/lrm: starting service vm:108
+info     21    node1/lrm: service status vm:108 started
+info     21    node1/lrm: starting service vm:109
+info     21    node1/lrm: service status vm:109 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     40    node1/crm: service 'vm:100': state changed from 'request_stop' to 'stopped'
+info    120      cmdlist: execute shutdown node1
+info    120    node1/lrm: got shutdown request with shutdown policy 'migrate'
+info    120    node1/lrm: shutdown LRM, doing maintenance, removing this node from active list
+info    120    node1/crm: node 'node1': state changed from 'online' => 'maintenance'
+info    120    node1/crm: migrate service 'vm:101' to node 'node2' (running)
+info    120    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:102' to node 'node2' (running)
+info    120    node1/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:103' to node 'node3' (running)
+info    120    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node1, target = node3)
+info    120    node1/crm: migrate service 'vm:104' to node 'node2' (running)
+info    120    node1/crm: service 'vm:104': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:105' to node 'node2' (running)
+info    120    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:106' to node 'node3' (running)
+info    120    node1/crm: service 'vm:106': state changed from 'started' to 'migrate'  (node = node1, target = node3)
+info    120    node1/crm: migrate service 'vm:107' to node 'node2' (running)
+info    120    node1/crm: service 'vm:107': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:108' to node 'node2' (running)
+info    120    node1/crm: service 'vm:108': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:109' to node 'node3' (running)
+info    120    node1/crm: service 'vm:109': state changed from 'started' to 'migrate'  (node = node1, target = node3)
+info    121    node1/lrm: status change active => maintenance
+info    121    node1/lrm: service vm:101 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:101 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:102 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:102 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:103 - start migrate to node 'node3'
+info    121    node1/lrm: service vm:103 - end migrate to node 'node3'
+info    121    node1/lrm: service vm:104 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:104 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:105 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:105 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:106 - start migrate to node 'node3'
+info    121    node1/lrm: service vm:106 - end migrate to node 'node3'
+info    121    node1/lrm: service vm:107 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:107 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:108 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:108 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:109 - start migrate to node 'node3'
+info    121    node1/lrm: service vm:109 - end migrate to node 'node3'
+info    140    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node3)
+info    140    node1/crm: service 'vm:104': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:106': state changed from 'migrate' to 'started'  (node = node3)
+info    140    node1/crm: service 'vm:107': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:108': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:109': state changed from 'migrate' to 'started'  (node = node3)
+info    142    node1/lrm: exit (loop end)
+info    142     shutdown: execute crm node1 stop
+info    141    node1/crm: server received shutdown request
+info    143    node2/lrm: got lock 'ha_agent_node2_lock'
+info    143    node2/lrm: status change wait_for_agent_lock => active
+info    143    node2/lrm: starting service vm:101
+info    143    node2/lrm: service status vm:101 started
+info    143    node2/lrm: starting service vm:102
+info    143    node2/lrm: service status vm:102 started
+info    143    node2/lrm: starting service vm:104
+info    143    node2/lrm: service status vm:104 started
+info    143    node2/lrm: starting service vm:105
+info    143    node2/lrm: service status vm:105 started
+info    143    node2/lrm: starting service vm:107
+info    143    node2/lrm: service status vm:107 started
+info    143    node2/lrm: starting service vm:108
+info    143    node2/lrm: service status vm:108 started
+info    145    node3/lrm: got lock 'ha_agent_node3_lock'
+info    145    node3/lrm: status change wait_for_agent_lock => active
+info    145    node3/lrm: starting service vm:103
+info    145    node3/lrm: service status vm:103 started
+info    145    node3/lrm: starting service vm:106
+info    145    node3/lrm: service status vm:106 started
+info    145    node3/lrm: starting service vm:109
+info    145    node3/lrm: service status vm:109 started
+info    160    node1/crm: voluntary release CRM lock
+info    161    node1/crm: exit (loop end)
+info    161     shutdown: execute power node1 off
+info    161    node2/crm: got lock 'ha_manager_lock'
+info    161    node2/crm: status change slave => master
+info    161    node2/crm: using scheduler mode 'static'
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-crs-static3/manager_status b/src/test/test-crs-static3/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static3/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static3/service_config b/src/test/test-crs-static3/service_config
new file mode 100644
index 0000000..47f94d3
--- /dev/null
+++ b/src/test/test-crs-static3/service_config
@@ -0,0 +1,12 @@
+{
+    "vm:100": { "node": "node1", "state": "stopped" },
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node1", "state": "enabled" },
+    "vm:103": { "node": "node1", "state": "enabled" },
+    "vm:104": { "node": "node1", "state": "enabled" },
+    "vm:105": { "node": "node1", "state": "enabled" },
+    "vm:106": { "node": "node1", "state": "enabled" },
+    "vm:107": { "node": "node1", "state": "enabled" },
+    "vm:108": { "node": "node1", "state": "enabled" },
+    "vm:109": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static3/static_service_stats b/src/test/test-crs-static3/static_service_stats
new file mode 100644
index 0000000..bca71cb
--- /dev/null
+++ b/src/test/test-crs-static3/static_service_stats
@@ -0,0 +1,12 @@
+{
+    "vm:100": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:101": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:102": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:103": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:104": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:105": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:106": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:107": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:108": { "maxcpu": 2, "maxmem": 4000000000 },
+    "vm:109": { "maxcpu": 2, "maxmem": 4000000000 }
+}
diff --git a/src/test/test-crs-static4/README b/src/test/test-crs-static4/README
new file mode 100644
index 0000000..4dfb1bc
--- /dev/null
+++ b/src/test/test-crs-static4/README
@@ -0,0 +1,6 @@
+Test how shutdown migrate policy works with the 'static' resource scheduling
+mode.
+
+Expect that, when node1 is shut down, the first service is migrated to node2 and
+all others to node 3, because the first service is very resource-heavy compared
+to the others.
diff --git a/src/test/test-crs-static4/cmdlist b/src/test/test-crs-static4/cmdlist
new file mode 100644
index 0000000..e84297f
--- /dev/null
+++ b/src/test/test-crs-static4/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "shutdown node1" ]
+]
diff --git a/src/test/test-crs-static4/datacenter.cfg b/src/test/test-crs-static4/datacenter.cfg
new file mode 100644
index 0000000..caa8148
--- /dev/null
+++ b/src/test/test-crs-static4/datacenter.cfg
@@ -0,0 +1,9 @@
+{
+    "crs": {
+        "ha": "static"
+    },
+    "ha": {
+        "shutdown_policy": "migrate"
+    }
+}
+
diff --git a/src/test/test-crs-static4/hardware_status b/src/test/test-crs-static4/hardware_status
new file mode 100644
index 0000000..a83a2dc
--- /dev/null
+++ b/src/test/test-crs-static4/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node3": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static4/log.expect b/src/test/test-crs-static4/log.expect
new file mode 100644
index 0000000..3eedc23
--- /dev/null
+++ b/src/test/test-crs-static4/log.expect
@@ -0,0 +1,149 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: using scheduler mode 'static'
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'ct:100' on node 'node1'
+info     20    node1/crm: adding new service 'ct:101' on node 'node1'
+info     20    node1/crm: adding new service 'ct:102' on node 'node1'
+info     20    node1/crm: adding new service 'ct:103' on node 'node1'
+info     20    node1/crm: adding new service 'ct:104' on node 'node1'
+info     20    node1/crm: adding new service 'ct:105' on node 'node1'
+info     20    node1/crm: adding new service 'ct:106' on node 'node1'
+info     20    node1/crm: adding new service 'ct:107' on node 'node1'
+info     20    node1/crm: adding new service 'ct:108' on node 'node1'
+info     20    node1/crm: adding new service 'ct:109' on node 'node1'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service ct:101
+info     21    node1/lrm: service status ct:101 started
+info     21    node1/lrm: starting service ct:102
+info     21    node1/lrm: service status ct:102 started
+info     21    node1/lrm: starting service ct:103
+info     21    node1/lrm: service status ct:103 started
+info     21    node1/lrm: starting service ct:104
+info     21    node1/lrm: service status ct:104 started
+info     21    node1/lrm: starting service ct:105
+info     21    node1/lrm: service status ct:105 started
+info     21    node1/lrm: starting service ct:106
+info     21    node1/lrm: service status ct:106 started
+info     21    node1/lrm: starting service ct:107
+info     21    node1/lrm: service status ct:107 started
+info     21    node1/lrm: starting service ct:108
+info     21    node1/lrm: service status ct:108 started
+info     21    node1/lrm: starting service ct:109
+info     21    node1/lrm: service status ct:109 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info     40    node1/crm: service 'ct:100': state changed from 'request_stop' to 'stopped'
+info    120      cmdlist: execute shutdown node1
+info    120    node1/lrm: got shutdown request with shutdown policy 'migrate'
+info    120    node1/lrm: shutdown LRM, doing maintenance, removing this node from active list
+info    120    node1/crm: node 'node1': state changed from 'online' => 'maintenance'
+info    120    node1/crm: relocate service 'ct:101' to node 'node2'
+info    120    node1/crm: service 'ct:101': state changed from 'started' to 'relocate'  (node = node1, target = node2)
+info    120    node1/crm: relocate service 'ct:102' to node 'node3'
+info    120    node1/crm: service 'ct:102': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:103' to node 'node3'
+info    120    node1/crm: service 'ct:103': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:104' to node 'node3'
+info    120    node1/crm: service 'ct:104': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:105' to node 'node3'
+info    120    node1/crm: service 'ct:105': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:106' to node 'node3'
+info    120    node1/crm: service 'ct:106': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:107' to node 'node3'
+info    120    node1/crm: service 'ct:107': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:108' to node 'node3'
+info    120    node1/crm: service 'ct:108': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:109' to node 'node3'
+info    120    node1/crm: service 'ct:109': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    121    node1/lrm: status change active => maintenance
+info    121    node1/lrm: service ct:101 - start relocate to node 'node2'
+info    121    node1/lrm: stopping service ct:101 (relocate)
+info    121    node1/lrm: service status ct:101 stopped
+info    121    node1/lrm: service ct:101 - end relocate to node 'node2'
+info    121    node1/lrm: service ct:102 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:102 (relocate)
+info    121    node1/lrm: service status ct:102 stopped
+info    121    node1/lrm: service ct:102 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:103 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:103 (relocate)
+info    121    node1/lrm: service status ct:103 stopped
+info    121    node1/lrm: service ct:103 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:104 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:104 (relocate)
+info    121    node1/lrm: service status ct:104 stopped
+info    121    node1/lrm: service ct:104 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:105 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:105 (relocate)
+info    121    node1/lrm: service status ct:105 stopped
+info    121    node1/lrm: service ct:105 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:106 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:106 (relocate)
+info    121    node1/lrm: service status ct:106 stopped
+info    121    node1/lrm: service ct:106 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:107 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:107 (relocate)
+info    121    node1/lrm: service status ct:107 stopped
+info    121    node1/lrm: service ct:107 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:108 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:108 (relocate)
+info    121    node1/lrm: service status ct:108 stopped
+info    121    node1/lrm: service ct:108 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:109 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:109 (relocate)
+info    121    node1/lrm: service status ct:109 stopped
+info    121    node1/lrm: service ct:109 - end relocate to node 'node3'
+info    140    node1/crm: service 'ct:101': state changed from 'relocate' to 'started'  (node = node2)
+info    140    node1/crm: service 'ct:102': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:103': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:104': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:105': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:106': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:107': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:108': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:109': state changed from 'relocate' to 'started'  (node = node3)
+info    142    node1/lrm: exit (loop end)
+info    142     shutdown: execute crm node1 stop
+info    141    node1/crm: server received shutdown request
+info    143    node2/lrm: got lock 'ha_agent_node2_lock'
+info    143    node2/lrm: status change wait_for_agent_lock => active
+info    143    node2/lrm: starting service ct:101
+info    143    node2/lrm: service status ct:101 started
+info    145    node3/lrm: got lock 'ha_agent_node3_lock'
+info    145    node3/lrm: status change wait_for_agent_lock => active
+info    145    node3/lrm: starting service ct:102
+info    145    node3/lrm: service status ct:102 started
+info    145    node3/lrm: starting service ct:103
+info    145    node3/lrm: service status ct:103 started
+info    145    node3/lrm: starting service ct:104
+info    145    node3/lrm: service status ct:104 started
+info    145    node3/lrm: starting service ct:105
+info    145    node3/lrm: service status ct:105 started
+info    145    node3/lrm: starting service ct:106
+info    145    node3/lrm: service status ct:106 started
+info    145    node3/lrm: starting service ct:107
+info    145    node3/lrm: service status ct:107 started
+info    145    node3/lrm: starting service ct:108
+info    145    node3/lrm: service status ct:108 started
+info    145    node3/lrm: starting service ct:109
+info    145    node3/lrm: service status ct:109 started
+info    160    node1/crm: voluntary release CRM lock
+info    161    node1/crm: exit (loop end)
+info    161     shutdown: execute power node1 off
+info    161    node2/crm: got lock 'ha_manager_lock'
+info    161    node2/crm: status change slave => master
+info    161    node2/crm: using scheduler mode 'static'
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-crs-static4/manager_status b/src/test/test-crs-static4/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static4/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static4/service_config b/src/test/test-crs-static4/service_config
new file mode 100644
index 0000000..b984a09
--- /dev/null
+++ b/src/test/test-crs-static4/service_config
@@ -0,0 +1,12 @@
+{
+    "ct:100": { "node": "node1", "state": "stopped" },
+    "ct:101": { "node": "node1", "state": "enabled" },
+    "ct:102": { "node": "node1", "state": "enabled" },
+    "ct:103": { "node": "node1", "state": "enabled" },
+    "ct:104": { "node": "node1", "state": "enabled" },
+    "ct:105": { "node": "node1", "state": "enabled" },
+    "ct:106": { "node": "node1", "state": "enabled" },
+    "ct:107": { "node": "node1", "state": "enabled" },
+    "ct:108": { "node": "node1", "state": "enabled" },
+    "ct:109": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static4/static_service_stats b/src/test/test-crs-static4/static_service_stats
new file mode 100644
index 0000000..878709b
--- /dev/null
+++ b/src/test/test-crs-static4/static_service_stats
@@ -0,0 +1,12 @@
+{
+    "ct:100": { "maxcpu": 2, "maxmem": 4000000000 },
+    "ct:101": { "maxcpu": 0, "maxmem": 40000000000 },
+    "ct:102": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:103": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:104": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:105": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:106": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:107": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:108": { "maxcpu": 2, "maxmem": 2000000000 },
+    "ct:109": { "maxcpu": 2, "maxmem": 2000000000 }
+}
diff --git a/src/test/test-crs-static5/README b/src/test/test-crs-static5/README
new file mode 100644
index 0000000..d9b5dc7
--- /dev/null
+++ b/src/test/test-crs-static5/README
@@ -0,0 +1,5 @@
+Test how recovery works with the 'static' resource scheduling mode.
+
+Expect that, when node1 is shut down, all services are migrated to node 3,
+because the services don't have much memory, node 2 and 3 both already have a
+service with high memory, but node 3 has much left-over CPU.
diff --git a/src/test/test-crs-static5/cmdlist b/src/test/test-crs-static5/cmdlist
new file mode 100644
index 0000000..e84297f
--- /dev/null
+++ b/src/test/test-crs-static5/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "shutdown node1" ]
+]
diff --git a/src/test/test-crs-static5/datacenter.cfg b/src/test/test-crs-static5/datacenter.cfg
new file mode 100644
index 0000000..caa8148
--- /dev/null
+++ b/src/test/test-crs-static5/datacenter.cfg
@@ -0,0 +1,9 @@
+{
+    "crs": {
+        "ha": "static"
+    },
+    "ha": {
+        "shutdown_policy": "migrate"
+    }
+}
+
diff --git a/src/test/test-crs-static5/hardware_status b/src/test/test-crs-static5/hardware_status
new file mode 100644
index 0000000..3eb9e73
--- /dev/null
+++ b/src/test/test-crs-static5/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node2": { "power": "off", "network": "off", "cpus": 32, "memory": 100000000000 },
+  "node3": { "power": "off", "network": "off", "cpus": 128, "memory": 100000000000 }
+}
diff --git a/src/test/test-crs-static5/log.expect b/src/test/test-crs-static5/log.expect
new file mode 100644
index 0000000..cb6b0d5
--- /dev/null
+++ b/src/test/test-crs-static5/log.expect
@@ -0,0 +1,117 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: using scheduler mode 'static'
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'ct:102' on node 'node2'
+info     20    node1/crm: adding new service 'ct:103' on node 'node3'
+info     20    node1/crm: adding new service 'ct:104' on node 'node1'
+info     20    node1/crm: adding new service 'ct:105' on node 'node1'
+info     20    node1/crm: adding new service 'ct:106' on node 'node1'
+info     20    node1/crm: adding new service 'ct:107' on node 'node1'
+info     20    node1/crm: adding new service 'ct:108' on node 'node1'
+info     20    node1/crm: adding new service 'ct:109' on node 'node1'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service ct:104
+info     21    node1/lrm: service status ct:104 started
+info     21    node1/lrm: starting service ct:105
+info     21    node1/lrm: service status ct:105 started
+info     21    node1/lrm: starting service ct:106
+info     21    node1/lrm: service status ct:106 started
+info     21    node1/lrm: starting service ct:107
+info     21    node1/lrm: service status ct:107 started
+info     21    node1/lrm: starting service ct:108
+info     21    node1/lrm: service status ct:108 started
+info     21    node1/lrm: starting service ct:109
+info     21    node1/lrm: service status ct:109 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service ct:102
+info     23    node2/lrm: service status ct:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service ct:103
+info     25    node3/lrm: service status ct:103 started
+info    120      cmdlist: execute shutdown node1
+info    120    node1/lrm: got shutdown request with shutdown policy 'migrate'
+info    120    node1/lrm: shutdown LRM, doing maintenance, removing this node from active list
+info    120    node1/crm: node 'node1': state changed from 'online' => 'maintenance'
+info    120    node1/crm: relocate service 'ct:104' to node 'node3'
+info    120    node1/crm: service 'ct:104': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:105' to node 'node3'
+info    120    node1/crm: service 'ct:105': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:106' to node 'node3'
+info    120    node1/crm: service 'ct:106': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:107' to node 'node3'
+info    120    node1/crm: service 'ct:107': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:108' to node 'node3'
+info    120    node1/crm: service 'ct:108': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    120    node1/crm: relocate service 'ct:109' to node 'node3'
+info    120    node1/crm: service 'ct:109': state changed from 'started' to 'relocate'  (node = node1, target = node3)
+info    121    node1/lrm: status change active => maintenance
+info    121    node1/lrm: service ct:104 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:104 (relocate)
+info    121    node1/lrm: service status ct:104 stopped
+info    121    node1/lrm: service ct:104 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:105 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:105 (relocate)
+info    121    node1/lrm: service status ct:105 stopped
+info    121    node1/lrm: service ct:105 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:106 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:106 (relocate)
+info    121    node1/lrm: service status ct:106 stopped
+info    121    node1/lrm: service ct:106 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:107 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:107 (relocate)
+info    121    node1/lrm: service status ct:107 stopped
+info    121    node1/lrm: service ct:107 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:108 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:108 (relocate)
+info    121    node1/lrm: service status ct:108 stopped
+info    121    node1/lrm: service ct:108 - end relocate to node 'node3'
+info    121    node1/lrm: service ct:109 - start relocate to node 'node3'
+info    121    node1/lrm: stopping service ct:109 (relocate)
+info    121    node1/lrm: service status ct:109 stopped
+info    121    node1/lrm: service ct:109 - end relocate to node 'node3'
+info    140    node1/crm: service 'ct:104': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:105': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:106': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:107': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:108': state changed from 'relocate' to 'started'  (node = node3)
+info    140    node1/crm: service 'ct:109': state changed from 'relocate' to 'started'  (node = node3)
+info    142    node1/lrm: exit (loop end)
+info    142     shutdown: execute crm node1 stop
+info    141    node1/crm: server received shutdown request
+info    145    node3/lrm: starting service ct:104
+info    145    node3/lrm: service status ct:104 started
+info    145    node3/lrm: starting service ct:105
+info    145    node3/lrm: service status ct:105 started
+info    145    node3/lrm: starting service ct:106
+info    145    node3/lrm: service status ct:106 started
+info    145    node3/lrm: starting service ct:107
+info    145    node3/lrm: service status ct:107 started
+info    145    node3/lrm: starting service ct:108
+info    145    node3/lrm: service status ct:108 started
+info    145    node3/lrm: starting service ct:109
+info    145    node3/lrm: service status ct:109 started
+info    160    node1/crm: voluntary release CRM lock
+info    161    node1/crm: exit (loop end)
+info    161     shutdown: execute power node1 off
+info    161    node2/crm: got lock 'ha_manager_lock'
+info    161    node2/crm: status change slave => master
+info    161    node2/crm: using scheduler mode 'static'
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-crs-static5/manager_status b/src/test/test-crs-static5/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-crs-static5/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-crs-static5/service_config b/src/test/test-crs-static5/service_config
new file mode 100644
index 0000000..43c5f60
--- /dev/null
+++ b/src/test/test-crs-static5/service_config
@@ -0,0 +1,10 @@
+{
+    "ct:102": { "node": "node2", "state": "enabled" },
+    "ct:103": { "node": "node3", "state": "enabled" },
+    "ct:104": { "node": "node1", "state": "enabled" },
+    "ct:105": { "node": "node1", "state": "enabled" },
+    "ct:106": { "node": "node1", "state": "enabled" },
+    "ct:107": { "node": "node1", "state": "enabled" },
+    "ct:108": { "node": "node1", "state": "enabled" },
+    "ct:109": { "node": "node1", "state": "enabled" }
+}
diff --git a/src/test/test-crs-static5/static_service_stats b/src/test/test-crs-static5/static_service_stats
new file mode 100644
index 0000000..6293f63
--- /dev/null
+++ b/src/test/test-crs-static5/static_service_stats
@@ -0,0 +1,11 @@
+{
+    "ct:101": { "maxcpu": 0,   "maxmem": 40000000000 },
+    "ct:102": { "maxcpu": 0.5, "maxmem": 40000000000 },
+    "ct:103": { "maxcpu": 0.5, "maxmem": 200000000 },
+    "ct:104": { "maxcpu": 0.5, "maxmem": 200000000 },
+    "ct:105": { "maxcpu": 0.5, "maxmem": 200000000 },
+    "ct:106": { "maxcpu": 0.5, "maxmem": 200000000 },
+    "ct:107": { "maxcpu": 0.5, "maxmem": 200000000 },
+    "ct:108": { "maxcpu": 0.5, "maxmem": 200000000 },
+    "ct:109": { "maxcpu": 0.5, "maxmem": 200000000 }
+}
-- 
2.30.2






More information about the pve-devel mailing list