[pve-devel] [PATCH pve-ha-manager 8/8] add tests
Alexandre Derumier
aderumier at odiso.com
Wed Apr 27 17:33:51 CEST 2022
---
src/test/test-basic0-balance-affinity/README | 1 +
src/test/test-basic0-balance-affinity/cmdlist | 3 ++
.../datacenter.cfg | 5 ++
.../hardware_status | 5 ++
.../test-basic0-balance-affinity/log.expect | 40 ++++++++++++++
.../manager_status | 1 +
.../test-basic0-balance-affinity/node_stats | 5 ++
.../resources_groups | 3 ++
.../service_config | 5 ++
.../service_stats | 5 ++
.../test-basic0-balance-cpupressure/README | 1 +
.../test-basic0-balance-cpupressure/cmdlist | 3 ++
.../datacenter.cfg | 5 ++
.../hardware_status | 5 ++
.../log.expect | 40 ++++++++++++++
.../manager_status | 1 +
.../node_stats | 5 ++
.../service_config | 5 ++
.../service_stats | 5 ++
src/test/test-basic0-recovery-affinity/README | 1 +
.../test-basic0-recovery-affinity/cmdlist | 4 ++
.../datacenter.cfg | 5 ++
.../hardware_status | 5 ++
.../test-basic0-recovery-affinity/log.expect | 53 +++++++++++++++++++
.../manager_status | 1 +
.../test-basic0-recovery-affinity/node_stats | 5 ++
.../resources_groups | 3 ++
.../service_config | 5 ++
.../service_stats | 5 ++
.../test-basic0-recovery-antifinity/README | 1 +
.../test-basic0-recovery-antifinity/cmdlist | 4 ++
.../datacenter.cfg | 5 ++
.../hardware_status | 5 ++
.../log.expect | 53 +++++++++++++++++++
.../manager_status | 1 +
.../node_stats | 5 ++
.../resources_groups | 3 ++
.../service_config | 5 ++
.../service_stats | 5 ++
src/test/test-basic00-recovery-storage/README | 1 +
.../test-basic00-recovery-storage/cmdlist | 4 ++
.../hardware_status | 5 ++
.../test-basic00-recovery-storage/log.expect | 52 ++++++++++++++++++
.../manager_status | 1 +
.../service_config | 5 ++
.../test-basic00-recovery-storage/storecfg | 3 ++
.../test-basic00-recovery-storage/vm_config | 5 ++
47 files changed, 393 insertions(+)
create mode 100644 src/test/test-basic0-balance-affinity/README
create mode 100644 src/test/test-basic0-balance-affinity/cmdlist
create mode 100644 src/test/test-basic0-balance-affinity/datacenter.cfg
create mode 100644 src/test/test-basic0-balance-affinity/hardware_status
create mode 100644 src/test/test-basic0-balance-affinity/log.expect
create mode 100644 src/test/test-basic0-balance-affinity/manager_status
create mode 100644 src/test/test-basic0-balance-affinity/node_stats
create mode 100644 src/test/test-basic0-balance-affinity/resources_groups
create mode 100644 src/test/test-basic0-balance-affinity/service_config
create mode 100644 src/test/test-basic0-balance-affinity/service_stats
create mode 100644 src/test/test-basic0-balance-cpupressure/README
create mode 100644 src/test/test-basic0-balance-cpupressure/cmdlist
create mode 100644 src/test/test-basic0-balance-cpupressure/datacenter.cfg
create mode 100644 src/test/test-basic0-balance-cpupressure/hardware_status
create mode 100644 src/test/test-basic0-balance-cpupressure/log.expect
create mode 100644 src/test/test-basic0-balance-cpupressure/manager_status
create mode 100644 src/test/test-basic0-balance-cpupressure/node_stats
create mode 100644 src/test/test-basic0-balance-cpupressure/service_config
create mode 100644 src/test/test-basic0-balance-cpupressure/service_stats
create mode 100644 src/test/test-basic0-recovery-affinity/README
create mode 100644 src/test/test-basic0-recovery-affinity/cmdlist
create mode 100644 src/test/test-basic0-recovery-affinity/datacenter.cfg
create mode 100644 src/test/test-basic0-recovery-affinity/hardware_status
create mode 100644 src/test/test-basic0-recovery-affinity/log.expect
create mode 100644 src/test/test-basic0-recovery-affinity/manager_status
create mode 100644 src/test/test-basic0-recovery-affinity/node_stats
create mode 100644 src/test/test-basic0-recovery-affinity/resources_groups
create mode 100644 src/test/test-basic0-recovery-affinity/service_config
create mode 100644 src/test/test-basic0-recovery-affinity/service_stats
create mode 100644 src/test/test-basic0-recovery-antifinity/README
create mode 100644 src/test/test-basic0-recovery-antifinity/cmdlist
create mode 100644 src/test/test-basic0-recovery-antifinity/datacenter.cfg
create mode 100644 src/test/test-basic0-recovery-antifinity/hardware_status
create mode 100644 src/test/test-basic0-recovery-antifinity/log.expect
create mode 100644 src/test/test-basic0-recovery-antifinity/manager_status
create mode 100644 src/test/test-basic0-recovery-antifinity/node_stats
create mode 100644 src/test/test-basic0-recovery-antifinity/resources_groups
create mode 100644 src/test/test-basic0-recovery-antifinity/service_config
create mode 100644 src/test/test-basic0-recovery-antifinity/service_stats
create mode 100644 src/test/test-basic00-recovery-storage/README
create mode 100644 src/test/test-basic00-recovery-storage/cmdlist
create mode 100644 src/test/test-basic00-recovery-storage/hardware_status
create mode 100644 src/test/test-basic00-recovery-storage/log.expect
create mode 100644 src/test/test-basic00-recovery-storage/manager_status
create mode 100644 src/test/test-basic00-recovery-storage/service_config
create mode 100644 src/test/test-basic00-recovery-storage/storecfg
create mode 100644 src/test/test-basic00-recovery-storage/vm_config
diff --git a/src/test/test-basic0-balance-affinity/README b/src/test/test-basic0-balance-affinity/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/cmdlist b/src/test/test-basic0-balance-affinity/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/cmdlist
@@ -0,0 +1,3 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-basic0-balance-affinity/datacenter.cfg b/src/test/test-basic0-balance-affinity/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+ "ha": {
+ "balancer": "1"
+ }
+}
diff --git a/src/test/test-basic0-balance-affinity/hardware_status b/src/test/test-basic0-balance-affinity/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/log.expect b/src/test/test-basic0-balance-affinity/log.expect
new file mode 100644
index 0000000..3f02d55
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/log.expect
@@ -0,0 +1,40 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 140 node1/crm: balancer - migrate service 'vm:101' to node 'node3' (running)
+info 140 node1/crm: service 'vm:101': state changed from 'started' to 'migrate' (node = node1, target = node3)
+info 141 node1/lrm: service vm:101 - start migrate to node 'node3'
+info 141 node1/lrm: service vm:101 - end migrate to node 'node3'
+info 160 node1/crm: service 'vm:101': state changed from 'migrate' to 'started' (node = node3)
+info 165 node3/lrm: starting service vm:101
+info 165 node3/lrm: service status vm:101 started
+info 620 hardware: exit simulation - done
diff --git a/src/test/test-basic0-balance-affinity/manager_status b/src/test/test-basic0-balance-affinity/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/node_stats b/src/test/test-basic0-balance-affinity/node_stats
new file mode 100644
index 0000000..967da45
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/node_stats
@@ -0,0 +1,5 @@
+{
+ "node1": { "cpu": 0.2, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+ "node2": { "cpu": 0.1, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+ "node3": { "cpu": 0.1, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/resources_groups b/src/test/test-basic0-balance-affinity/resources_groups
new file mode 100644
index 0000000..7f1927d
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/resources_groups
@@ -0,0 +1,3 @@
+resourcegroup: group1
+ resources vm:101,vm:103
+ affinity group
diff --git a/src/test/test-basic0-balance-affinity/service_config b/src/test/test-basic0-balance-affinity/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2", "state": "enabled" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/service_stats b/src/test/test-basic0-balance-affinity/service_stats
new file mode 100644
index 0000000..2806fed
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/service_stats
@@ -0,0 +1,5 @@
+{
+ "101": { "cpu": 0.5, "maxcpu": 16,"mem": 107374182,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/README b/src/test/test-basic0-balance-cpupressure/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/cmdlist b/src/test/test-basic0-balance-cpupressure/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/cmdlist
@@ -0,0 +1,3 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-basic0-balance-cpupressure/datacenter.cfg b/src/test/test-basic0-balance-cpupressure/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+ "ha": {
+ "balancer": "1"
+ }
+}
diff --git a/src/test/test-basic0-balance-cpupressure/hardware_status b/src/test/test-basic0-balance-cpupressure/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/log.expect b/src/test/test-basic0-balance-cpupressure/log.expect
new file mode 100644
index 0000000..0087761
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/log.expect
@@ -0,0 +1,40 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 140 node1/crm: balancer - migrate service 'vm:102' to node 'node1' (running)
+info 140 node1/crm: service 'vm:102': state changed from 'started' to 'migrate' (node = node2, target = node1)
+info 143 node2/lrm: service vm:102 - start migrate to node 'node1'
+info 143 node2/lrm: service vm:102 - end migrate to node 'node1'
+info 160 node1/crm: service 'vm:102': state changed from 'migrate' to 'started' (node = node1)
+info 161 node1/lrm: starting service vm:102
+info 161 node1/lrm: service status vm:102 started
+info 620 hardware: exit simulation - done
diff --git a/src/test/test-basic0-balance-cpupressure/manager_status b/src/test/test-basic0-balance-cpupressure/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/node_stats b/src/test/test-basic0-balance-cpupressure/node_stats
new file mode 100644
index 0000000..285da50
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/node_stats
@@ -0,0 +1,5 @@
+{
+ "node1": { "cpu": 0.1, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+ "node2": { "cpu": 0.3, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+ "node3": { "cpu": 0.7, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/service_config b/src/test/test-basic0-balance-cpupressure/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2", "state": "enabled" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/service_stats b/src/test/test-basic0-balance-cpupressure/service_stats
new file mode 100644
index 0000000..babdb01
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/service_stats
@@ -0,0 +1,5 @@
+{
+ "101": { "cpu": 0.5, "maxcpu": 16,"mem": 107374182,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 10 },
+ "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/README b/src/test/test-basic0-recovery-affinity/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/cmdlist b/src/test/test-basic0-recovery-affinity/cmdlist
new file mode 100644
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "network node3 off" ]
+]
diff --git a/src/test/test-basic0-recovery-affinity/datacenter.cfg b/src/test/test-basic0-recovery-affinity/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+ "ha": {
+ "balancer": "1"
+ }
+}
diff --git a/src/test/test-basic0-recovery-affinity/hardware_status b/src/test/test-basic0-recovery-affinity/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/log.expect b/src/test/test-basic0-recovery-affinity/log.expect
new file mode 100644
index 0000000..31ded22
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/log.expect
@@ -0,0 +1,53 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 120 cmdlist: execute network node3 off
+info 120 node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info 124 node3/crm: status change slave => wait_for_quorum
+info 125 node3/lrm: status change active => lost_agent_lock
+info 160 node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info 160 node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai 160 node1/crm: FENCE: Try to fence node 'node3'
+info 166 watchdog: execute power node3 off
+info 165 node3/crm: killed by poweroff
+info 166 node3/lrm: killed by poweroff
+info 166 hardware: server 'node3' stopped by poweroff (watchdog)
+info 240 node1/crm: got lock 'ha_agent_node3_lock'
+info 240 node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info 240 node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai 240 node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info 240 node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info 240 node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
+info 240 node1/crm: service 'vm:103': state changed from 'recovery' to 'started' (node = node1)
+info 241 node1/lrm: starting service vm:103
+info 241 node1/lrm: service status vm:103 started
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-basic0-recovery-affinity/manager_status b/src/test/test-basic0-recovery-affinity/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/node_stats b/src/test/test-basic0-recovery-affinity/node_stats
new file mode 100644
index 0000000..967da45
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/node_stats
@@ -0,0 +1,5 @@
+{
+ "node1": { "cpu": 0.2, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+ "node2": { "cpu": 0.1, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+ "node3": { "cpu": 0.1, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/resources_groups b/src/test/test-basic0-recovery-affinity/resources_groups
new file mode 100644
index 0000000..7f1927d
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/resources_groups
@@ -0,0 +1,3 @@
+resourcegroup: group1
+ resources vm:101,vm:103
+ affinity group
diff --git a/src/test/test-basic0-recovery-affinity/service_config b/src/test/test-basic0-recovery-affinity/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2", "state": "enabled" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/service_stats b/src/test/test-basic0-recovery-affinity/service_stats
new file mode 100644
index 0000000..46ed27e
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/service_stats
@@ -0,0 +1,5 @@
+{
+ "101": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/README b/src/test/test-basic0-recovery-antifinity/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/cmdlist b/src/test/test-basic0-recovery-antifinity/cmdlist
new file mode 100644
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "network node3 off" ]
+]
diff --git a/src/test/test-basic0-recovery-antifinity/datacenter.cfg b/src/test/test-basic0-recovery-antifinity/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+ "ha": {
+ "balancer": "1"
+ }
+}
diff --git a/src/test/test-basic0-recovery-antifinity/hardware_status b/src/test/test-basic0-recovery-antifinity/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/log.expect b/src/test/test-basic0-recovery-antifinity/log.expect
new file mode 100644
index 0000000..31ded22
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/log.expect
@@ -0,0 +1,53 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 120 cmdlist: execute network node3 off
+info 120 node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info 124 node3/crm: status change slave => wait_for_quorum
+info 125 node3/lrm: status change active => lost_agent_lock
+info 160 node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info 160 node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai 160 node1/crm: FENCE: Try to fence node 'node3'
+info 166 watchdog: execute power node3 off
+info 165 node3/crm: killed by poweroff
+info 166 node3/lrm: killed by poweroff
+info 166 hardware: server 'node3' stopped by poweroff (watchdog)
+info 240 node1/crm: got lock 'ha_agent_node3_lock'
+info 240 node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info 240 node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai 240 node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info 240 node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info 240 node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
+info 240 node1/crm: service 'vm:103': state changed from 'recovery' to 'started' (node = node1)
+info 241 node1/lrm: starting service vm:103
+info 241 node1/lrm: service status vm:103 started
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-basic0-recovery-antifinity/manager_status b/src/test/test-basic0-recovery-antifinity/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/node_stats b/src/test/test-basic0-recovery-antifinity/node_stats
new file mode 100644
index 0000000..967da45
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/node_stats
@@ -0,0 +1,5 @@
+{
+ "node1": { "cpu": 0.2, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+ "node2": { "cpu": 0.1, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+ "node3": { "cpu": 0.1, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/resources_groups b/src/test/test-basic0-recovery-antifinity/resources_groups
new file mode 100644
index 0000000..af3b211
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/resources_groups
@@ -0,0 +1,3 @@
+resourcegroup: group1
+ resources vm:102,vm:103
+ affinity separate
diff --git a/src/test/test-basic0-recovery-antifinity/service_config b/src/test/test-basic0-recovery-antifinity/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2", "state": "enabled" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/service_stats b/src/test/test-basic0-recovery-antifinity/service_stats
new file mode 100644
index 0000000..46ed27e
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/service_stats
@@ -0,0 +1,5 @@
+{
+ "101": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+ "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/README b/src/test/test-basic00-recovery-storage/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/cmdlist b/src/test/test-basic00-recovery-storage/cmdlist
new file mode 100644
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/cmdlist
@@ -0,0 +1,4 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"],
+ [ "network node3 off" ]
+]
diff --git a/src/test/test-basic00-recovery-storage/hardware_status b/src/test/test-basic00-recovery-storage/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off" },
+ "node2": { "power": "off", "network": "off" },
+ "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/log.expect b/src/test/test-basic00-recovery-storage/log.expect
new file mode 100644
index 0000000..3302fcc
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/log.expect
@@ -0,0 +1,52 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 40 node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info 120 cmdlist: execute network node3 off
+info 120 node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info 124 node3/crm: status change slave => wait_for_quorum
+info 125 node3/lrm: status change active => lost_agent_lock
+info 160 node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info 160 node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai 160 node1/crm: FENCE: Try to fence node 'node3'
+info 166 watchdog: execute power node3 off
+info 165 node3/crm: killed by poweroff
+info 166 node3/lrm: killed by poweroff
+info 166 hardware: server 'node3' stopped by poweroff (watchdog)
+info 240 node1/crm: got lock 'ha_agent_node3_lock'
+info 240 node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info 240 node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai 240 node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info 240 node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info 240 node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
+info 240 node1/crm: service 'vm:103': state changed from 'recovery' to 'started' (node = node1)
+info 241 node1/lrm: starting service vm:103
+info 241 node1/lrm: service status vm:103 started
+info 720 hardware: exit simulation - done
diff --git a/src/test/test-basic00-recovery-storage/manager_status b/src/test/test-basic00-recovery-storage/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/service_config b/src/test/test-basic00-recovery-storage/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "enabled" },
+ "vm:102": { "node": "node2" },
+ "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/storecfg b/src/test/test-basic00-recovery-storage/storecfg
new file mode 100644
index 0000000..9cd641c
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/storecfg
@@ -0,0 +1,3 @@
+{
+ "local": { "nodes": { "node1": 1, "node3": 1 } }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/vm_config b/src/test/test-basic00-recovery-storage/vm_config
new file mode 100644
index 0000000..d6f2d7d
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/vm_config
@@ -0,0 +1,5 @@
+{
+ "101": { "storage": "local" },
+ "102": { "storage": "local" },
+ "103": { "storage": "local" }
+}
\ No newline at end of file
--
2.30.2
More information about the pve-devel
mailing list