[pve-devel] [PATCH pve-ha-manager 8/8] add tests

Alexandre Derumier aderumier at odiso.com
Wed Apr 27 17:33:51 CEST 2022


---
 src/test/test-basic0-balance-affinity/README  |  1 +
 src/test/test-basic0-balance-affinity/cmdlist |  3 ++
 .../datacenter.cfg                            |  5 ++
 .../hardware_status                           |  5 ++
 .../test-basic0-balance-affinity/log.expect   | 40 ++++++++++++++
 .../manager_status                            |  1 +
 .../test-basic0-balance-affinity/node_stats   |  5 ++
 .../resources_groups                          |  3 ++
 .../service_config                            |  5 ++
 .../service_stats                             |  5 ++
 .../test-basic0-balance-cpupressure/README    |  1 +
 .../test-basic0-balance-cpupressure/cmdlist   |  3 ++
 .../datacenter.cfg                            |  5 ++
 .../hardware_status                           |  5 ++
 .../log.expect                                | 40 ++++++++++++++
 .../manager_status                            |  1 +
 .../node_stats                                |  5 ++
 .../service_config                            |  5 ++
 .../service_stats                             |  5 ++
 src/test/test-basic0-recovery-affinity/README |  1 +
 .../test-basic0-recovery-affinity/cmdlist     |  4 ++
 .../datacenter.cfg                            |  5 ++
 .../hardware_status                           |  5 ++
 .../test-basic0-recovery-affinity/log.expect  | 53 +++++++++++++++++++
 .../manager_status                            |  1 +
 .../test-basic0-recovery-affinity/node_stats  |  5 ++
 .../resources_groups                          |  3 ++
 .../service_config                            |  5 ++
 .../service_stats                             |  5 ++
 .../test-basic0-recovery-antifinity/README    |  1 +
 .../test-basic0-recovery-antifinity/cmdlist   |  4 ++
 .../datacenter.cfg                            |  5 ++
 .../hardware_status                           |  5 ++
 .../log.expect                                | 53 +++++++++++++++++++
 .../manager_status                            |  1 +
 .../node_stats                                |  5 ++
 .../resources_groups                          |  3 ++
 .../service_config                            |  5 ++
 .../service_stats                             |  5 ++
 src/test/test-basic00-recovery-storage/README |  1 +
 .../test-basic00-recovery-storage/cmdlist     |  4 ++
 .../hardware_status                           |  5 ++
 .../test-basic00-recovery-storage/log.expect  | 52 ++++++++++++++++++
 .../manager_status                            |  1 +
 .../service_config                            |  5 ++
 .../test-basic00-recovery-storage/storecfg    |  3 ++
 .../test-basic00-recovery-storage/vm_config   |  5 ++
 47 files changed, 393 insertions(+)
 create mode 100644 src/test/test-basic0-balance-affinity/README
 create mode 100644 src/test/test-basic0-balance-affinity/cmdlist
 create mode 100644 src/test/test-basic0-balance-affinity/datacenter.cfg
 create mode 100644 src/test/test-basic0-balance-affinity/hardware_status
 create mode 100644 src/test/test-basic0-balance-affinity/log.expect
 create mode 100644 src/test/test-basic0-balance-affinity/manager_status
 create mode 100644 src/test/test-basic0-balance-affinity/node_stats
 create mode 100644 src/test/test-basic0-balance-affinity/resources_groups
 create mode 100644 src/test/test-basic0-balance-affinity/service_config
 create mode 100644 src/test/test-basic0-balance-affinity/service_stats
 create mode 100644 src/test/test-basic0-balance-cpupressure/README
 create mode 100644 src/test/test-basic0-balance-cpupressure/cmdlist
 create mode 100644 src/test/test-basic0-balance-cpupressure/datacenter.cfg
 create mode 100644 src/test/test-basic0-balance-cpupressure/hardware_status
 create mode 100644 src/test/test-basic0-balance-cpupressure/log.expect
 create mode 100644 src/test/test-basic0-balance-cpupressure/manager_status
 create mode 100644 src/test/test-basic0-balance-cpupressure/node_stats
 create mode 100644 src/test/test-basic0-balance-cpupressure/service_config
 create mode 100644 src/test/test-basic0-balance-cpupressure/service_stats
 create mode 100644 src/test/test-basic0-recovery-affinity/README
 create mode 100644 src/test/test-basic0-recovery-affinity/cmdlist
 create mode 100644 src/test/test-basic0-recovery-affinity/datacenter.cfg
 create mode 100644 src/test/test-basic0-recovery-affinity/hardware_status
 create mode 100644 src/test/test-basic0-recovery-affinity/log.expect
 create mode 100644 src/test/test-basic0-recovery-affinity/manager_status
 create mode 100644 src/test/test-basic0-recovery-affinity/node_stats
 create mode 100644 src/test/test-basic0-recovery-affinity/resources_groups
 create mode 100644 src/test/test-basic0-recovery-affinity/service_config
 create mode 100644 src/test/test-basic0-recovery-affinity/service_stats
 create mode 100644 src/test/test-basic0-recovery-antifinity/README
 create mode 100644 src/test/test-basic0-recovery-antifinity/cmdlist
 create mode 100644 src/test/test-basic0-recovery-antifinity/datacenter.cfg
 create mode 100644 src/test/test-basic0-recovery-antifinity/hardware_status
 create mode 100644 src/test/test-basic0-recovery-antifinity/log.expect
 create mode 100644 src/test/test-basic0-recovery-antifinity/manager_status
 create mode 100644 src/test/test-basic0-recovery-antifinity/node_stats
 create mode 100644 src/test/test-basic0-recovery-antifinity/resources_groups
 create mode 100644 src/test/test-basic0-recovery-antifinity/service_config
 create mode 100644 src/test/test-basic0-recovery-antifinity/service_stats
 create mode 100644 src/test/test-basic00-recovery-storage/README
 create mode 100644 src/test/test-basic00-recovery-storage/cmdlist
 create mode 100644 src/test/test-basic00-recovery-storage/hardware_status
 create mode 100644 src/test/test-basic00-recovery-storage/log.expect
 create mode 100644 src/test/test-basic00-recovery-storage/manager_status
 create mode 100644 src/test/test-basic00-recovery-storage/service_config
 create mode 100644 src/test/test-basic00-recovery-storage/storecfg
 create mode 100644 src/test/test-basic00-recovery-storage/vm_config

diff --git a/src/test/test-basic0-balance-affinity/README b/src/test/test-basic0-balance-affinity/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/cmdlist b/src/test/test-basic0-balance-affinity/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/cmdlist
@@ -0,0 +1,3 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-basic0-balance-affinity/datacenter.cfg b/src/test/test-basic0-balance-affinity/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+    "ha": {
+        "balancer": "1"
+    }
+}
diff --git a/src/test/test-basic0-balance-affinity/hardware_status b/src/test/test-basic0-balance-affinity/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/log.expect b/src/test/test-basic0-balance-affinity/log.expect
new file mode 100644
index 0000000..3f02d55
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/log.expect
@@ -0,0 +1,40 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    140    node1/crm: balancer - migrate service 'vm:101' to node 'node3' (running)
+info    140    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (node = node1, target = node3)
+info    141    node1/lrm: service vm:101 - start migrate to node 'node3'
+info    141    node1/lrm: service vm:101 - end migrate to node 'node3'
+info    160    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node3)
+info    165    node3/lrm: starting service vm:101
+info    165    node3/lrm: service status vm:101 started
+info    620     hardware: exit simulation - done
diff --git a/src/test/test-basic0-balance-affinity/manager_status b/src/test/test-basic0-balance-affinity/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/node_stats b/src/test/test-basic0-balance-affinity/node_stats
new file mode 100644
index 0000000..967da45
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/node_stats
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "cpu": 0.2, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+  "node2": { "cpu": 0.1, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+  "node3": { "cpu": 0.1, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/resources_groups b/src/test/test-basic0-balance-affinity/resources_groups
new file mode 100644
index 0000000..7f1927d
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/resources_groups
@@ -0,0 +1,3 @@
+resourcegroup: group1
+        resources vm:101,vm:103
+        affinity group
diff --git a/src/test/test-basic0-balance-affinity/service_config b/src/test/test-basic0-balance-affinity/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2", "state": "enabled" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-affinity/service_stats b/src/test/test-basic0-balance-affinity/service_stats
new file mode 100644
index 0000000..2806fed
--- /dev/null
+++ b/src/test/test-basic0-balance-affinity/service_stats
@@ -0,0 +1,5 @@
+{ 
+  "101": { "cpu": 0.5, "maxcpu": 16,"mem": 107374182,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/README b/src/test/test-basic0-balance-cpupressure/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/cmdlist b/src/test/test-basic0-balance-cpupressure/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/cmdlist
@@ -0,0 +1,3 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-basic0-balance-cpupressure/datacenter.cfg b/src/test/test-basic0-balance-cpupressure/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+    "ha": {
+        "balancer": "1"
+    }
+}
diff --git a/src/test/test-basic0-balance-cpupressure/hardware_status b/src/test/test-basic0-balance-cpupressure/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/log.expect b/src/test/test-basic0-balance-cpupressure/log.expect
new file mode 100644
index 0000000..0087761
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/log.expect
@@ -0,0 +1,40 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    140    node1/crm: balancer - migrate service 'vm:102' to node 'node1' (running)
+info    140    node1/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node2, target = node1)
+info    143    node2/lrm: service vm:102 - start migrate to node 'node1'
+info    143    node2/lrm: service vm:102 - end migrate to node 'node1'
+info    160    node1/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node1)
+info    161    node1/lrm: starting service vm:102
+info    161    node1/lrm: service status vm:102 started
+info    620     hardware: exit simulation - done
diff --git a/src/test/test-basic0-balance-cpupressure/manager_status b/src/test/test-basic0-balance-cpupressure/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/node_stats b/src/test/test-basic0-balance-cpupressure/node_stats
new file mode 100644
index 0000000..285da50
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/node_stats
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "cpu": 0.1, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+  "node2": { "cpu": 0.3, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+  "node3": { "cpu": 0.7, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/service_config b/src/test/test-basic0-balance-cpupressure/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2", "state": "enabled" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-balance-cpupressure/service_stats b/src/test/test-basic0-balance-cpupressure/service_stats
new file mode 100644
index 0000000..babdb01
--- /dev/null
+++ b/src/test/test-basic0-balance-cpupressure/service_stats
@@ -0,0 +1,5 @@
+{ 
+  "101": { "cpu": 0.5, "maxcpu": 16,"mem": 107374182,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 10 },
+  "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/README b/src/test/test-basic0-recovery-affinity/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/cmdlist b/src/test/test-basic0-recovery-affinity/cmdlist
new file mode 100644
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-basic0-recovery-affinity/datacenter.cfg b/src/test/test-basic0-recovery-affinity/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+    "ha": {
+        "balancer": "1"
+    }
+}
diff --git a/src/test/test-basic0-recovery-affinity/hardware_status b/src/test/test-basic0-recovery-affinity/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/log.expect b/src/test/test-basic0-recovery-affinity/log.expect
new file mode 100644
index 0000000..31ded22
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/log.expect
@@ -0,0 +1,53 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
+info    241    node1/lrm: starting service vm:103
+info    241    node1/lrm: service status vm:103 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-basic0-recovery-affinity/manager_status b/src/test/test-basic0-recovery-affinity/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/node_stats b/src/test/test-basic0-recovery-affinity/node_stats
new file mode 100644
index 0000000..967da45
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/node_stats
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "cpu": 0.2, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+  "node2": { "cpu": 0.1, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+  "node3": { "cpu": 0.1, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/resources_groups b/src/test/test-basic0-recovery-affinity/resources_groups
new file mode 100644
index 0000000..7f1927d
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/resources_groups
@@ -0,0 +1,3 @@
+resourcegroup: group1
+        resources vm:101,vm:103
+        affinity group
diff --git a/src/test/test-basic0-recovery-affinity/service_config b/src/test/test-basic0-recovery-affinity/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2", "state": "enabled" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-affinity/service_stats b/src/test/test-basic0-recovery-affinity/service_stats
new file mode 100644
index 0000000..46ed27e
--- /dev/null
+++ b/src/test/test-basic0-recovery-affinity/service_stats
@@ -0,0 +1,5 @@
+{ 
+  "101": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/README b/src/test/test-basic0-recovery-antifinity/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/cmdlist b/src/test/test-basic0-recovery-antifinity/cmdlist
new file mode 100644
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-basic0-recovery-antifinity/datacenter.cfg b/src/test/test-basic0-recovery-antifinity/datacenter.cfg
new file mode 100644
index 0000000..3de5c2a
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/datacenter.cfg
@@ -0,0 +1,5 @@
+{
+    "ha": {
+        "balancer": "1"
+    }
+}
diff --git a/src/test/test-basic0-recovery-antifinity/hardware_status b/src/test/test-basic0-recovery-antifinity/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/log.expect b/src/test/test-basic0-recovery-antifinity/log.expect
new file mode 100644
index 0000000..31ded22
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/log.expect
@@ -0,0 +1,53 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
+info    241    node1/lrm: starting service vm:103
+info    241    node1/lrm: service status vm:103 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-basic0-recovery-antifinity/manager_status b/src/test/test-basic0-recovery-antifinity/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/node_stats b/src/test/test-basic0-recovery-antifinity/node_stats
new file mode 100644
index 0000000..967da45
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/node_stats
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "cpu": 0.2, "maxcpu": 32,"mem": 20737418239,"maxmem": 107374182400 },
+  "node2": { "cpu": 0.1, "maxcpu": 32,"mem": 22737418240,"maxmem": 107374182400 },
+  "node3": { "cpu": 0.1, "maxcpu": 32,"mem": 10737418240,"maxmem": 107374182400 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/resources_groups b/src/test/test-basic0-recovery-antifinity/resources_groups
new file mode 100644
index 0000000..af3b211
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/resources_groups
@@ -0,0 +1,3 @@
+resourcegroup: group1
+        resources vm:102,vm:103
+        affinity separate
diff --git a/src/test/test-basic0-recovery-antifinity/service_config b/src/test/test-basic0-recovery-antifinity/service_config
new file mode 100644
index 0000000..c202a34
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2", "state": "enabled" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic0-recovery-antifinity/service_stats b/src/test/test-basic0-recovery-antifinity/service_stats
new file mode 100644
index 0000000..46ed27e
--- /dev/null
+++ b/src/test/test-basic0-recovery-antifinity/service_stats
@@ -0,0 +1,5 @@
+{ 
+  "101": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "102": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 },
+  "103": { "cpu": 0.5, "maxcpu": 16,"mem": 1073741824,"maxmem": 1073741824, "cpu_pressure": 0 }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/README b/src/test/test-basic00-recovery-storage/README
new file mode 100644
index 0000000..223c9dc
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/README
@@ -0,0 +1 @@
+Test failover after single node network failure.
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/cmdlist b/src/test/test-basic00-recovery-storage/cmdlist
new file mode 100644
index 0000000..eee0e40
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-basic00-recovery-storage/hardware_status b/src/test/test-basic00-recovery-storage/hardware_status
new file mode 100644
index 0000000..119b81c
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/hardware_status
@@ -0,0 +1,5 @@
+{ 
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/log.expect b/src/test/test-basic00-recovery-storage/log.expect
new file mode 100644
index 0000000..3302fcc
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/log.expect
@@ -0,0 +1,52 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info     40    node1/crm: service 'vm:102': state changed from 'request_stop' to 'stopped'
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
+info    241    node1/lrm: starting service vm:103
+info    241    node1/lrm: service status vm:103 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-basic00-recovery-storage/manager_status b/src/test/test-basic00-recovery-storage/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/service_config b/src/test/test-basic00-recovery-storage/service_config
new file mode 100644
index 0000000..0e05ab4
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "enabled" },
+    "vm:102": { "node": "node2" },
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/storecfg b/src/test/test-basic00-recovery-storage/storecfg
new file mode 100644
index 0000000..9cd641c
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/storecfg
@@ -0,0 +1,3 @@
+{ 
+  "local": { "nodes": { "node1": 1, "node3": 1 } }
+}
\ No newline at end of file
diff --git a/src/test/test-basic00-recovery-storage/vm_config b/src/test/test-basic00-recovery-storage/vm_config
new file mode 100644
index 0000000..d6f2d7d
--- /dev/null
+++ b/src/test/test-basic00-recovery-storage/vm_config
@@ -0,0 +1,5 @@
+{ 
+  "101": { "storage": "local" },
+  "102": { "storage": "local" },
+  "103": { "storage": "local" }
+}
\ No newline at end of file
-- 
2.30.2





More information about the pve-devel mailing list