[pve-devel] [RFC ha-manager 3/3] test: ha tester: add ha groups migration tests with runtime upgrades

Daniel Kral d.kral at proxmox.com
Thu Jul 31 16:27:19 CEST 2025


These test cases cover slightly more realistic upgrade paths of the
cluster nodes, where nodes are upgraded and rebooted one-by-one and some
actions might fail in-between.

The new sim_hardware_cmd 'version' is introduced to allow simulating the
runtime upgrades of each node and should be removed as soon as the HA
groups migration support code is not needed anymore (e.g. PVE 10).

Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
These test cases depend on the previous two patches, because the first
one tests an upgrade from 7.4-4 to 8.4.1 and the second test case
upgrades from 8.4.1 to 9.0.0 directly (instead of 9.0.0~16 as in the
other test cases).

 src/PVE/HA/Sim/Hardware.pm                   |   8 +
 src/test/test-group-migrate3/README          |   7 +
 src/test/test-group-migrate3/cmdlist         |  17 +
 src/test/test-group-migrate3/groups          |   7 +
 src/test/test-group-migrate3/hardware_status |   5 +
 src/test/test-group-migrate3/log.expect      | 344 +++++++++++++++++++
 src/test/test-group-migrate3/manager_status  |   1 +
 src/test/test-group-migrate3/service_config  |   5 +
 src/test/test-group-migrate4/README          |   8 +
 src/test/test-group-migrate4/cmdlist         |  15 +
 src/test/test-group-migrate4/groups          |   7 +
 src/test/test-group-migrate4/hardware_status |   5 +
 src/test/test-group-migrate4/log.expect      | 277 +++++++++++++++
 src/test/test-group-migrate4/manager_status  |   1 +
 src/test/test-group-migrate4/service_config  |   5 +
 15 files changed, 712 insertions(+)
 create mode 100644 src/test/test-group-migrate3/README
 create mode 100644 src/test/test-group-migrate3/cmdlist
 create mode 100644 src/test/test-group-migrate3/groups
 create mode 100644 src/test/test-group-migrate3/hardware_status
 create mode 100644 src/test/test-group-migrate3/log.expect
 create mode 100644 src/test/test-group-migrate3/manager_status
 create mode 100644 src/test/test-group-migrate3/service_config
 create mode 100644 src/test/test-group-migrate4/README
 create mode 100644 src/test/test-group-migrate4/cmdlist
 create mode 100644 src/test/test-group-migrate4/groups
 create mode 100644 src/test/test-group-migrate4/hardware_status
 create mode 100644 src/test/test-group-migrate4/log.expect
 create mode 100644 src/test/test-group-migrate4/manager_status
 create mode 100644 src/test/test-group-migrate4/service_config

diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index 4207ce31..63eb89ff 100644
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -596,6 +596,7 @@ sub get_cfs_state {
 # simulate hardware commands, the following commands are available:
 #   power <node> <on|off>
 #   network <node> <on|off>
+#   version <node> set <version>
 #   delay <seconds>
 #   skip-round <crm|lrm> [<rounds=1>]
 #   cfs <node> <rw|update> <work|fail>
@@ -683,6 +684,13 @@ sub sim_hardware_cmd {
 
             $self->write_hardware_status_nolock($cstatus);
 
+        } elsif ($cmd eq 'version') {
+            die "sim_hardware_cmd: unknown version action '$action'"
+                if $action ne "set";
+            $cstatus->{$node}->{version} = $param;
+
+            $self->write_hardware_status_nolock($cstatus);
+
         } elsif ($cmd eq 'cfs') {
             die "sim_hardware_cmd: unknown cfs action '$action' for node '$node'"
                 if $action !~ m/^(rw|update)$/;
diff --git a/src/test/test-group-migrate3/README b/src/test/test-group-migrate3/README
new file mode 100644
index 00000000..0ee45f7a
--- /dev/null
+++ b/src/test/test-group-migrate3/README
@@ -0,0 +1,7 @@
+Test whether an initial (unsupported) mixed version cluster can be properly
+upgraded per major version and then the CRM correctly migrates the HA group
+config only after all nodes have at least the proper pre-release version.
+
+By rebooting every node after each version change, this tests whether the
+switching of the CRM node and a few instances of LRM restarts are properly
+prohibiting the HA groups config migration.
diff --git a/src/test/test-group-migrate3/cmdlist b/src/test/test-group-migrate3/cmdlist
new file mode 100644
index 00000000..d507acad
--- /dev/null
+++ b/src/test/test-group-migrate3/cmdlist
@@ -0,0 +1,17 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "version node1 set 8.4.1" ],
+    [ "reboot node1" ],
+    [ "version node2 set 8.4.1" ],
+    [ "reboot node2" ],
+    [ "version node3 set 8.4.1" ],
+    [ "reboot node3" ],
+    [ "version node1 set 9.0.0~16" ],
+    [ "reboot node1" ],
+    [ "version node2 set 9.0.0~16" ],
+    [ "reboot node2" ],
+    [ "version node3 set 9.0.0~15" ],
+    [ "reboot node3" ],
+    [ "version node3 set 9.0.0~17" ],
+    [ "reboot node3" ]
+]
diff --git a/src/test/test-group-migrate3/groups b/src/test/test-group-migrate3/groups
new file mode 100644
index 00000000..bad746ca
--- /dev/null
+++ b/src/test/test-group-migrate3/groups
@@ -0,0 +1,7 @@
+group: group1
+	nodes node1
+	restricted 1
+
+group: group2
+	nodes node2:2,node3
+	nofailback 1
diff --git a/src/test/test-group-migrate3/hardware_status b/src/test/test-group-migrate3/hardware_status
new file mode 100644
index 00000000..e8f9d73f
--- /dev/null
+++ b/src/test/test-group-migrate3/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "version": "7.4-4" },
+  "node2": { "power": "off", "network": "off", "version": "8.3.7" },
+  "node3": { "power": "off", "network": "off", "version": "8.3.0" }
+}
diff --git a/src/test/test-group-migrate3/log.expect b/src/test/test-group-migrate3/log.expect
new file mode 100644
index 00000000..63be1218
--- /dev/null
+++ b/src/test/test-group-migrate3/log.expect
@@ -0,0 +1,344 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node2)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+noti     60    node1/crm: start ha group migration...
+noti     60    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti     60    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti     60    node1/crm: ha groups migration: node 'node1' has version '7.4-4'
+err      60    node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err      60    node1/crm: ha groups migration failed
+noti     60    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    120      cmdlist: execute version node1 set 8.4.1
+noti    180    node1/crm: start ha group migration...
+noti    180    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti    180    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    180    node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     180    node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     180    node1/crm: ha groups migration failed
+noti    180    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    220      cmdlist: execute reboot node1
+info    220    node1/lrm: got shutdown request with shutdown policy 'conditional'
+info    220    node1/lrm: reboot LRM, stop and freeze all services
+info    220    node1/crm: service 'vm:101': state changed from 'started' to 'freeze'
+info    221    node1/lrm: stopping service vm:101
+info    221    node1/lrm: service status vm:101 stopped
+info    222    node1/lrm: exit (loop end)
+info    222       reboot: execute crm node1 stop
+info    221    node1/crm: server received shutdown request
+info    240    node1/crm: voluntary release CRM lock
+info    241    node1/crm: exit (loop end)
+info    241       reboot: execute power node1 off
+info    241       reboot: execute power node1 on
+info    241    node1/crm: status change startup => wait_for_quorum
+info    240    node1/lrm: status change startup => wait_for_agent_lock
+info    242    node2/crm: got lock 'ha_manager_lock'
+info    242    node2/crm: status change slave => master
+info    242    node2/crm: service 'vm:101': state changed from 'freeze' to 'started'
+info    260    node1/crm: status change wait_for_quorum => slave
+info    261    node1/lrm: got lock 'ha_agent_node1_lock'
+info    261    node1/lrm: status change wait_for_agent_lock => active
+info    261    node1/lrm: starting service vm:101
+info    261    node1/lrm: service status vm:101 started
+noti    282    node2/crm: start ha group migration...
+noti    282    node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti    282    node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    282    node2/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     282    node2/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     282    node2/crm: ha groups migration failed
+noti    282    node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    320      cmdlist: execute version node2 set 8.4.1
+noti    402    node2/crm: start ha group migration...
+noti    402    node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti    402    node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    402    node2/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     402    node2/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     402    node2/crm: ha groups migration failed
+noti    402    node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    420      cmdlist: execute reboot node2
+info    420    node2/lrm: got shutdown request with shutdown policy 'conditional'
+info    420    node2/lrm: reboot LRM, stop and freeze all services
+info    422    node2/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info    423    node2/lrm: stopping service vm:102
+info    423    node2/lrm: service status vm:102 stopped
+info    424    node2/lrm: exit (loop end)
+info    424       reboot: execute crm node2 stop
+info    423    node2/crm: server received shutdown request
+info    442    node2/crm: voluntary release CRM lock
+info    443    node2/crm: exit (loop end)
+info    443       reboot: execute power node2 off
+info    443       reboot: execute power node2 on
+info    443    node2/crm: status change startup => wait_for_quorum
+info    440    node2/lrm: status change startup => wait_for_agent_lock
+info    444    node3/crm: got lock 'ha_manager_lock'
+info    444    node3/crm: status change slave => master
+info    444    node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info    462    node2/crm: status change wait_for_quorum => slave
+info    463    node2/lrm: got lock 'ha_agent_node2_lock'
+info    463    node2/lrm: status change wait_for_agent_lock => active
+info    463    node2/lrm: starting service vm:102
+info    463    node2/lrm: service status vm:102 started
+noti    484    node3/crm: start ha group migration...
+noti    484    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti    484    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    484    node3/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     484    node3/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     484    node3/crm: ha groups migration failed
+noti    484    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    520      cmdlist: execute version node3 set 8.4.1
+noti    604    node3/crm: start ha group migration...
+noti    604    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti    604    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    604    node3/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     604    node3/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     604    node3/crm: ha groups migration failed
+noti    604    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    620      cmdlist: execute reboot node3
+info    620    node3/lrm: got shutdown request with shutdown policy 'conditional'
+info    620    node3/lrm: reboot LRM, stop and freeze all services
+info    624    node3/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info    625    node3/lrm: stopping service vm:103
+info    625    node3/lrm: service status vm:103 stopped
+info    626    node3/lrm: exit (loop end)
+info    626       reboot: execute crm node3 stop
+info    625    node3/crm: server received shutdown request
+info    644    node3/crm: voluntary release CRM lock
+info    645    node3/crm: exit (loop end)
+info    645       reboot: execute power node3 off
+info    645       reboot: execute power node3 on
+info    645    node3/crm: status change startup => wait_for_quorum
+info    640    node3/lrm: status change startup => wait_for_agent_lock
+info    660    node1/crm: got lock 'ha_manager_lock'
+info    660    node1/crm: status change slave => master
+info    660    node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info    664    node3/crm: status change wait_for_quorum => slave
+info    665    node3/lrm: got lock 'ha_agent_node3_lock'
+info    665    node3/lrm: status change wait_for_agent_lock => active
+info    665    node3/lrm: starting service vm:103
+info    665    node3/lrm: service status vm:103 started
+noti    700    node1/crm: start ha group migration...
+noti    700    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti    700    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    700    node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     700    node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     700    node1/crm: ha groups migration failed
+noti    700    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    720      cmdlist: execute version node1 set 9.0.0~16
+info    820      cmdlist: execute reboot node1
+info    820    node1/lrm: got shutdown request with shutdown policy 'conditional'
+info    820    node1/lrm: reboot LRM, stop and freeze all services
+noti    820    node1/crm: start ha group migration...
+noti    820    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti    820    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'restart'
+err     820    node1/crm: abort ha groups migration: lrm 'node1' is not in mode 'active'
+err     820    node1/crm: ha groups migration failed
+noti    820    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    820    node1/crm: service 'vm:101': state changed from 'started' to 'freeze'
+info    821    node1/lrm: stopping service vm:101
+info    821    node1/lrm: service status vm:101 stopped
+info    822    node1/lrm: exit (loop end)
+info    822       reboot: execute crm node1 stop
+info    821    node1/crm: server received shutdown request
+info    840    node1/crm: voluntary release CRM lock
+info    841    node1/crm: exit (loop end)
+info    841       reboot: execute power node1 off
+info    841       reboot: execute power node1 on
+info    841    node1/crm: status change startup => wait_for_quorum
+info    840    node1/lrm: status change startup => wait_for_agent_lock
+info    842    node2/crm: got lock 'ha_manager_lock'
+info    842    node2/crm: status change slave => master
+info    842    node2/crm: service 'vm:101': state changed from 'freeze' to 'started'
+info    860    node1/crm: status change wait_for_quorum => slave
+info    861    node1/lrm: got lock 'ha_agent_node1_lock'
+info    861    node1/lrm: status change wait_for_agent_lock => active
+info    861    node1/lrm: starting service vm:101
+info    861    node1/lrm: service status vm:101 started
+noti    882    node2/crm: start ha group migration...
+noti    882    node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti    882    node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    882    node2/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti    882    node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti    882    node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti    882    node2/crm: ha groups migration: node 'node2' has version '8.4.1'
+err     882    node2/crm: abort ha groups migration: node 'node2' needs at least pve-manager version '9.0.0~16'
+err     882    node2/crm: ha groups migration failed
+noti    882    node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    920      cmdlist: execute version node2 set 9.0.0~16
+noti   1002    node2/crm: start ha group migration...
+noti   1002    node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1002    node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1002    node2/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti   1002    node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1002    node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1002    node2/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti   1002    node2/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1002    node2/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1002    node2/crm: ha groups migration: node 'node3' has version '8.4.1'
+err    1002    node2/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err    1002    node2/crm: ha groups migration failed
+noti   1002    node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1020      cmdlist: execute reboot node2
+info   1020    node2/lrm: got shutdown request with shutdown policy 'conditional'
+info   1020    node2/lrm: reboot LRM, stop and freeze all services
+info   1022    node2/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info   1023    node2/lrm: stopping service vm:102
+info   1023    node2/lrm: service status vm:102 stopped
+info   1024    node2/lrm: exit (loop end)
+info   1024       reboot: execute crm node2 stop
+info   1023    node2/crm: server received shutdown request
+info   1042    node2/crm: voluntary release CRM lock
+info   1043    node2/crm: exit (loop end)
+info   1043       reboot: execute power node2 off
+info   1043       reboot: execute power node2 on
+info   1043    node2/crm: status change startup => wait_for_quorum
+info   1040    node2/lrm: status change startup => wait_for_agent_lock
+info   1044    node3/crm: got lock 'ha_manager_lock'
+info   1044    node3/crm: status change slave => master
+info   1044    node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info   1062    node2/crm: status change wait_for_quorum => slave
+info   1063    node2/lrm: got lock 'ha_agent_node2_lock'
+info   1063    node2/lrm: status change wait_for_agent_lock => active
+info   1063    node2/lrm: starting service vm:102
+info   1063    node2/lrm: service status vm:102 started
+noti   1084    node3/crm: start ha group migration...
+noti   1084    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1084    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1084    node3/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti   1084    node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1084    node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1084    node3/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti   1084    node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1084    node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1084    node3/crm: ha groups migration: node 'node3' has version '8.4.1'
+err    1084    node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err    1084    node3/crm: ha groups migration failed
+noti   1084    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1120      cmdlist: execute version node3 set 9.0.0~15
+noti   1204    node3/crm: start ha group migration...
+noti   1204    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1204    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1204    node3/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti   1204    node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1204    node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1204    node3/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti   1204    node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1204    node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1204    node3/crm: ha groups migration: node 'node3' has version '9.0.0~15'
+err    1204    node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err    1204    node3/crm: ha groups migration failed
+noti   1204    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1220      cmdlist: execute reboot node3
+info   1220    node3/lrm: got shutdown request with shutdown policy 'conditional'
+info   1220    node3/lrm: reboot LRM, stop and freeze all services
+info   1224    node3/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info   1225    node3/lrm: stopping service vm:103
+info   1225    node3/lrm: service status vm:103 stopped
+info   1226    node3/lrm: exit (loop end)
+info   1226       reboot: execute crm node3 stop
+info   1225    node3/crm: server received shutdown request
+info   1244    node3/crm: voluntary release CRM lock
+info   1245    node3/crm: exit (loop end)
+info   1245       reboot: execute power node3 off
+info   1245       reboot: execute power node3 on
+info   1245    node3/crm: status change startup => wait_for_quorum
+info   1240    node3/lrm: status change startup => wait_for_agent_lock
+info   1260    node1/crm: got lock 'ha_manager_lock'
+info   1260    node1/crm: status change slave => master
+info   1260    node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info   1264    node3/crm: status change wait_for_quorum => slave
+info   1265    node3/lrm: got lock 'ha_agent_node3_lock'
+info   1265    node3/lrm: status change wait_for_agent_lock => active
+info   1265    node3/lrm: starting service vm:103
+info   1265    node3/lrm: service status vm:103 started
+noti   1300    node1/crm: start ha group migration...
+noti   1300    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1300    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1300    node1/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti   1300    node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1300    node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1300    node1/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti   1300    node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1300    node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1300    node1/crm: ha groups migration: node 'node3' has version '9.0.0~15'
+err    1300    node1/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err    1300    node1/crm: ha groups migration failed
+noti   1300    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1320      cmdlist: execute version node3 set 9.0.0~17
+info   1420      cmdlist: execute reboot node3
+info   1420    node3/lrm: got shutdown request with shutdown policy 'conditional'
+info   1420    node3/lrm: reboot LRM, stop and freeze all services
+noti   1420    node1/crm: start ha group migration...
+noti   1420    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1420    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1420    node1/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti   1420    node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1420    node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1420    node1/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti   1420    node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1420    node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'restart'
+err    1420    node1/crm: abort ha groups migration: lrm 'node3' is not in mode 'active'
+err    1420    node1/crm: ha groups migration failed
+noti   1420    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1420    node1/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info   1425    node3/lrm: stopping service vm:103
+info   1425    node3/lrm: service status vm:103 stopped
+info   1426    node3/lrm: exit (loop end)
+info   1426       reboot: execute crm node3 stop
+info   1425    node3/crm: server received shutdown request
+info   1445    node3/crm: exit (loop end)
+info   1445       reboot: execute power node3 off
+info   1445       reboot: execute power node3 on
+info   1445    node3/crm: status change startup => wait_for_quorum
+info   1440    node3/lrm: status change startup => wait_for_agent_lock
+info   1460    node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info   1464    node3/crm: status change wait_for_quorum => slave
+info   1465    node3/lrm: got lock 'ha_agent_node3_lock'
+info   1465    node3/lrm: status change wait_for_agent_lock => active
+info   1465    node3/lrm: starting service vm:103
+info   1465    node3/lrm: service status vm:103 started
+noti   1540    node1/crm: start ha group migration...
+noti   1540    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1540    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1540    node1/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti   1540    node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1540    node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1540    node1/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti   1540    node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1540    node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1540    node1/crm: ha groups migration: node 'node3' has version '9.0.0~17'
+noti   1540    node1/crm: ha groups migration: migration to rules config successful
+noti   1540    node1/crm: ha groups migration: migration to resources config successful
+noti   1540    node1/crm: ha groups migration: group config deletion successful
+noti   1540    node1/crm: ha groups migration successful
+info   2020     hardware: exit simulation - done
diff --git a/src/test/test-group-migrate3/manager_status b/src/test/test-group-migrate3/manager_status
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/test/test-group-migrate3/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-group-migrate3/service_config b/src/test/test-group-migrate3/service_config
new file mode 100644
index 00000000..a27551e5
--- /dev/null
+++ b/src/test/test-group-migrate3/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "started", "group": "group1" },
+    "vm:102": { "node": "node2", "state": "started", "group": "group2" },
+    "vm:103": { "node": "node3", "state": "started", "group": "group2" }
+}
diff --git a/src/test/test-group-migrate4/README b/src/test/test-group-migrate4/README
new file mode 100644
index 00000000..37e60c7d
--- /dev/null
+++ b/src/test/test-group-migrate4/README
@@ -0,0 +1,8 @@
+Test whether a cluster, where all nodes have the same version from the previous
+major release, can be properly upgraded to the neede major release version and
+then the CRM correctly migrates the HA group config only after all nodes have
+the minimum major release version.
+
+Additionally, the nodes are rebooted with every version upgrade and in-between
+the CFS sporadically fails to read/write, fails to update cluster state and an
+LRM is restarted, which all prohibit the HA groups config migration.
diff --git a/src/test/test-group-migrate4/cmdlist b/src/test/test-group-migrate4/cmdlist
new file mode 100644
index 00000000..fdd3bfdd
--- /dev/null
+++ b/src/test/test-group-migrate4/cmdlist
@@ -0,0 +1,15 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "delay 10" ],
+    [ "version node1 set 9.0.0" ],
+    [ "reboot node1" ],
+    [ "cfs node2 rw fail" ],
+    [ "version node2 set 9.0.0" ],
+    [ "cfs node2 rw work" ],
+    [ "reboot node2" ],
+    [ "cfs node3 update fail" ],
+    [ "cfs node3 update work" ],
+    [ "version node3 set 9.0.1" ],
+    [ "restart-lrm node2" ],
+    [ "reboot node3" ]
+]
diff --git a/src/test/test-group-migrate4/groups b/src/test/test-group-migrate4/groups
new file mode 100644
index 00000000..bad746ca
--- /dev/null
+++ b/src/test/test-group-migrate4/groups
@@ -0,0 +1,7 @@
+group: group1
+	nodes node1
+	restricted 1
+
+group: group2
+	nodes node2:2,node3
+	nofailback 1
diff --git a/src/test/test-group-migrate4/hardware_status b/src/test/test-group-migrate4/hardware_status
new file mode 100644
index 00000000..7ad46416
--- /dev/null
+++ b/src/test/test-group-migrate4/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off", "version": "8.4.1" },
+  "node2": { "power": "off", "network": "off", "version": "8.4.1" },
+  "node3": { "power": "off", "network": "off", "version": "8.4.1" }
+}
diff --git a/src/test/test-group-migrate4/log.expect b/src/test/test-group-migrate4/log.expect
new file mode 100644
index 00000000..7ffe33e3
--- /dev/null
+++ b/src/test/test-group-migrate4/log.expect
@@ -0,0 +1,277 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node2'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node2)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:102
+info     23    node2/lrm: service status vm:102 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+noti     60    node1/crm: start ha group migration...
+noti     60    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti     60    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti     60    node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err      60    node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err      60    node1/crm: ha groups migration failed
+noti     60    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    120      cmdlist: execute delay 10
+noti    180    node1/crm: start ha group migration...
+noti    180    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti    180    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    180    node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err     180    node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err     180    node1/crm: ha groups migration failed
+noti    180    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    220      cmdlist: execute version node1 set 9.0.0
+noti    300    node1/crm: start ha group migration...
+noti    300    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti    300    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    300    node1/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti    300    node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti    300    node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti    300    node1/crm: ha groups migration: node 'node2' has version '8.4.1'
+err     300    node1/crm: abort ha groups migration: node 'node2' needs at least pve-manager version '9.0.0~16'
+err     300    node1/crm: ha groups migration failed
+noti    300    node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    320      cmdlist: execute reboot node1
+info    320    node1/lrm: got shutdown request with shutdown policy 'conditional'
+info    320    node1/lrm: reboot LRM, stop and freeze all services
+info    320    node1/crm: service 'vm:101': state changed from 'started' to 'freeze'
+info    321    node1/lrm: stopping service vm:101
+info    321    node1/lrm: service status vm:101 stopped
+info    322    node1/lrm: exit (loop end)
+info    322       reboot: execute crm node1 stop
+info    321    node1/crm: server received shutdown request
+info    340    node1/crm: voluntary release CRM lock
+info    341    node1/crm: exit (loop end)
+info    341       reboot: execute power node1 off
+info    341       reboot: execute power node1 on
+info    341    node1/crm: status change startup => wait_for_quorum
+info    340    node1/lrm: status change startup => wait_for_agent_lock
+info    342    node2/crm: got lock 'ha_manager_lock'
+info    342    node2/crm: status change slave => master
+info    342    node2/crm: service 'vm:101': state changed from 'freeze' to 'started'
+info    360    node1/crm: status change wait_for_quorum => slave
+info    361    node1/lrm: got lock 'ha_agent_node1_lock'
+info    361    node1/lrm: status change wait_for_agent_lock => active
+info    361    node1/lrm: starting service vm:101
+info    361    node1/lrm: service status vm:101 started
+noti    382    node2/crm: start ha group migration...
+noti    382    node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti    382    node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    382    node2/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti    382    node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti    382    node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti    382    node2/crm: ha groups migration: node 'node2' has version '8.4.1'
+err     382    node2/crm: abort ha groups migration: node 'node2' needs at least pve-manager version '9.0.0~16'
+err     382    node2/crm: ha groups migration failed
+noti    382    node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    420      cmdlist: execute cfs node2 rw fail
+err     422    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     422    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     423    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     423    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     423    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     442    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     442    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     443    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     443    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     443    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     462    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     462    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     463    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     463    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     463    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     482    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     482    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     483    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     483    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     483    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     502    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     502    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     503    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     503    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     503    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+info    520      cmdlist: execute version node2 set 9.0.0
+err     522    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     522    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     523    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     523    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     523    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     542    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     542    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     543    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     543    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     543    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     562    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     562    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     563    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     563    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     563    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     582    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     582    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     583    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     583    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     583    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err     602    node2/crm: could not read manager status: cfs connection refused - not mounted?
+err     602    node2/crm: got unexpected error - cfs connection refused - not mounted?
+err     603    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     603    node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err     603    node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+info    620      cmdlist: execute cfs node2 rw work
+noti    702    node2/crm: start ha group migration...
+noti    702    node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti    702    node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    702    node2/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti    702    node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti    702    node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti    702    node2/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti    702    node2/crm: ha groups migration: node 'node3' is in state 'online'
+noti    702    node2/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti    702    node2/crm: ha groups migration: node 'node3' has version '8.4.1'
+err     702    node2/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err     702    node2/crm: ha groups migration failed
+noti    702    node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    720      cmdlist: execute reboot node2
+info    720    node2/lrm: got shutdown request with shutdown policy 'conditional'
+info    720    node2/lrm: reboot LRM, stop and freeze all services
+info    722    node2/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info    723    node2/lrm: stopping service vm:102
+info    723    node2/lrm: service status vm:102 stopped
+info    724    node2/lrm: exit (loop end)
+info    724       reboot: execute crm node2 stop
+info    723    node2/crm: server received shutdown request
+info    742    node2/crm: voluntary release CRM lock
+info    743    node2/crm: exit (loop end)
+info    743       reboot: execute power node2 off
+info    743       reboot: execute power node2 on
+info    743    node2/crm: status change startup => wait_for_quorum
+info    740    node2/lrm: status change startup => wait_for_agent_lock
+info    744    node3/crm: got lock 'ha_manager_lock'
+info    744    node3/crm: status change slave => master
+info    744    node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info    762    node2/crm: status change wait_for_quorum => slave
+info    763    node2/lrm: got lock 'ha_agent_node2_lock'
+info    763    node2/lrm: status change wait_for_agent_lock => active
+info    763    node2/lrm: starting service vm:102
+info    763    node2/lrm: service status vm:102 started
+noti    784    node3/crm: start ha group migration...
+noti    784    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti    784    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti    784    node3/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti    784    node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti    784    node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti    784    node3/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti    784    node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti    784    node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti    784    node3/crm: ha groups migration: node 'node3' has version '8.4.1'
+err     784    node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err     784    node3/crm: ha groups migration failed
+noti    784    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info    820      cmdlist: execute cfs node3 update fail
+noti    824    node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    825    node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    844    node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    845    node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    864    node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    865    node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    884    node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    885    node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    904    node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti    905    node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+info    920      cmdlist: execute cfs node3 update work
+noti   1004    node3/crm: start ha group migration...
+noti   1004    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1004    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1004    node3/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti   1004    node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1004    node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1004    node3/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti   1004    node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1004    node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1004    node3/crm: ha groups migration: node 'node3' has version '8.4.1'
+err    1004    node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err    1004    node3/crm: ha groups migration failed
+noti   1004    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1020      cmdlist: execute version node3 set 9.0.1
+info   1120      cmdlist: execute restart-lrm node2
+info   1120    node2/lrm: restart LRM, freeze all services
+noti   1124    node3/crm: start ha group migration...
+noti   1124    node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1124    node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1124    node3/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti   1124    node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1124    node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'restart'
+err    1124    node3/crm: abort ha groups migration: lrm 'node2' is not in mode 'active'
+err    1124    node3/crm: ha groups migration failed
+noti   1124    node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info   1124    node3/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info   1144    node2/lrm: exit (loop end)
+info   1144    node2/lrm: status change startup => wait_for_agent_lock
+info   1164    node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info   1183    node2/lrm: got lock 'ha_agent_node2_lock'
+info   1183    node2/lrm: status change wait_for_agent_lock => active
+info   1220      cmdlist: execute reboot node3
+info   1220    node3/lrm: got shutdown request with shutdown policy 'conditional'
+info   1220    node3/lrm: reboot LRM, stop and freeze all services
+info   1224    node3/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info   1225    node3/lrm: stopping service vm:103
+info   1225    node3/lrm: service status vm:103 stopped
+info   1226    node3/lrm: exit (loop end)
+info   1226       reboot: execute crm node3 stop
+info   1225    node3/crm: server received shutdown request
+info   1244    node3/crm: voluntary release CRM lock
+info   1245    node3/crm: exit (loop end)
+info   1245       reboot: execute power node3 off
+info   1245       reboot: execute power node3 on
+info   1245    node3/crm: status change startup => wait_for_quorum
+info   1240    node3/lrm: status change startup => wait_for_agent_lock
+info   1260    node1/crm: got lock 'ha_manager_lock'
+info   1260    node1/crm: status change slave => master
+info   1260    node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info   1264    node3/crm: status change wait_for_quorum => slave
+info   1265    node3/lrm: got lock 'ha_agent_node3_lock'
+info   1265    node3/lrm: status change wait_for_agent_lock => active
+info   1265    node3/lrm: starting service vm:103
+info   1265    node3/lrm: service status vm:103 started
+noti   1300    node1/crm: start ha group migration...
+noti   1300    node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti   1300    node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti   1300    node1/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti   1300    node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti   1300    node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti   1300    node1/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti   1300    node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti   1300    node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti   1300    node1/crm: ha groups migration: node 'node3' has version '9.0.1'
+noti   1300    node1/crm: ha groups migration: migration to rules config successful
+noti   1300    node1/crm: ha groups migration: migration to resources config successful
+noti   1300    node1/crm: ha groups migration: group config deletion successful
+noti   1300    node1/crm: ha groups migration successful
+info   1820     hardware: exit simulation - done
diff --git a/src/test/test-group-migrate4/manager_status b/src/test/test-group-migrate4/manager_status
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/test/test-group-migrate4/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-group-migrate4/service_config b/src/test/test-group-migrate4/service_config
new file mode 100644
index 00000000..a27551e5
--- /dev/null
+++ b/src/test/test-group-migrate4/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "started", "group": "group1" },
+    "vm:102": { "node": "node2", "state": "started", "group": "group2" },
+    "vm:103": { "node": "node3", "state": "started", "group": "group2" }
+}
-- 
2.47.2





More information about the pve-devel mailing list