[pve-devel] [RFC ha-manager 3/3] test: ha tester: add ha groups migration tests with runtime upgrades
Daniel Kral
d.kral at proxmox.com
Thu Jul 31 16:27:19 CEST 2025
These test cases cover slightly more realistic upgrade paths of the
cluster nodes, where nodes are upgraded and rebooted one-by-one and some
actions might fail in-between.
The new sim_hardware_cmd 'version' is introduced to allow simulating the
runtime upgrades of each node and should be removed as soon as the HA
groups migration support code is not needed anymore (e.g. PVE 10).
Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
These test cases depend on the previous two patches, because the first
one tests an upgrade from 7.4-4 to 8.4.1 and the second test case
upgrades from 8.4.1 to 9.0.0 directly (instead of 9.0.0~16 as in the
other test cases).
src/PVE/HA/Sim/Hardware.pm | 8 +
src/test/test-group-migrate3/README | 7 +
src/test/test-group-migrate3/cmdlist | 17 +
src/test/test-group-migrate3/groups | 7 +
src/test/test-group-migrate3/hardware_status | 5 +
src/test/test-group-migrate3/log.expect | 344 +++++++++++++++++++
src/test/test-group-migrate3/manager_status | 1 +
src/test/test-group-migrate3/service_config | 5 +
src/test/test-group-migrate4/README | 8 +
src/test/test-group-migrate4/cmdlist | 15 +
src/test/test-group-migrate4/groups | 7 +
src/test/test-group-migrate4/hardware_status | 5 +
src/test/test-group-migrate4/log.expect | 277 +++++++++++++++
src/test/test-group-migrate4/manager_status | 1 +
src/test/test-group-migrate4/service_config | 5 +
15 files changed, 712 insertions(+)
create mode 100644 src/test/test-group-migrate3/README
create mode 100644 src/test/test-group-migrate3/cmdlist
create mode 100644 src/test/test-group-migrate3/groups
create mode 100644 src/test/test-group-migrate3/hardware_status
create mode 100644 src/test/test-group-migrate3/log.expect
create mode 100644 src/test/test-group-migrate3/manager_status
create mode 100644 src/test/test-group-migrate3/service_config
create mode 100644 src/test/test-group-migrate4/README
create mode 100644 src/test/test-group-migrate4/cmdlist
create mode 100644 src/test/test-group-migrate4/groups
create mode 100644 src/test/test-group-migrate4/hardware_status
create mode 100644 src/test/test-group-migrate4/log.expect
create mode 100644 src/test/test-group-migrate4/manager_status
create mode 100644 src/test/test-group-migrate4/service_config
diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index 4207ce31..63eb89ff 100644
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -596,6 +596,7 @@ sub get_cfs_state {
# simulate hardware commands, the following commands are available:
# power <node> <on|off>
# network <node> <on|off>
+# version <node> set <version>
# delay <seconds>
# skip-round <crm|lrm> [<rounds=1>]
# cfs <node> <rw|update> <work|fail>
@@ -683,6 +684,13 @@ sub sim_hardware_cmd {
$self->write_hardware_status_nolock($cstatus);
+ } elsif ($cmd eq 'version') {
+ die "sim_hardware_cmd: unknown version action '$action'"
+ if $action ne "set";
+ $cstatus->{$node}->{version} = $param;
+
+ $self->write_hardware_status_nolock($cstatus);
+
} elsif ($cmd eq 'cfs') {
die "sim_hardware_cmd: unknown cfs action '$action' for node '$node'"
if $action !~ m/^(rw|update)$/;
diff --git a/src/test/test-group-migrate3/README b/src/test/test-group-migrate3/README
new file mode 100644
index 00000000..0ee45f7a
--- /dev/null
+++ b/src/test/test-group-migrate3/README
@@ -0,0 +1,7 @@
+Test whether an initial (unsupported) mixed version cluster can be properly
+upgraded per major version and then the CRM correctly migrates the HA group
+config only after all nodes have at least the proper pre-release version.
+
+By rebooting every node after each version change, this tests whether the
+switching of the CRM node and a few instances of LRM restarts are properly
+prohibiting the HA groups config migration.
diff --git a/src/test/test-group-migrate3/cmdlist b/src/test/test-group-migrate3/cmdlist
new file mode 100644
index 00000000..d507acad
--- /dev/null
+++ b/src/test/test-group-migrate3/cmdlist
@@ -0,0 +1,17 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on" ],
+ [ "version node1 set 8.4.1" ],
+ [ "reboot node1" ],
+ [ "version node2 set 8.4.1" ],
+ [ "reboot node2" ],
+ [ "version node3 set 8.4.1" ],
+ [ "reboot node3" ],
+ [ "version node1 set 9.0.0~16" ],
+ [ "reboot node1" ],
+ [ "version node2 set 9.0.0~16" ],
+ [ "reboot node2" ],
+ [ "version node3 set 9.0.0~15" ],
+ [ "reboot node3" ],
+ [ "version node3 set 9.0.0~17" ],
+ [ "reboot node3" ]
+]
diff --git a/src/test/test-group-migrate3/groups b/src/test/test-group-migrate3/groups
new file mode 100644
index 00000000..bad746ca
--- /dev/null
+++ b/src/test/test-group-migrate3/groups
@@ -0,0 +1,7 @@
+group: group1
+ nodes node1
+ restricted 1
+
+group: group2
+ nodes node2:2,node3
+ nofailback 1
diff --git a/src/test/test-group-migrate3/hardware_status b/src/test/test-group-migrate3/hardware_status
new file mode 100644
index 00000000..e8f9d73f
--- /dev/null
+++ b/src/test/test-group-migrate3/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "version": "7.4-4" },
+ "node2": { "power": "off", "network": "off", "version": "8.3.7" },
+ "node3": { "power": "off", "network": "off", "version": "8.3.0" }
+}
diff --git a/src/test/test-group-migrate3/log.expect b/src/test/test-group-migrate3/log.expect
new file mode 100644
index 00000000..63be1218
--- /dev/null
+++ b/src/test/test-group-migrate3/log.expect
@@ -0,0 +1,344 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:101': state changed from 'request_start' to 'started' (node = node1)
+info 20 node1/crm: service 'vm:102': state changed from 'request_start' to 'started' (node = node2)
+info 20 node1/crm: service 'vm:103': state changed from 'request_start' to 'started' (node = node3)
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+noti 60 node1/crm: start ha group migration...
+noti 60 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 60 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 60 node1/crm: ha groups migration: node 'node1' has version '7.4-4'
+err 60 node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 60 node1/crm: ha groups migration failed
+noti 60 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 120 cmdlist: execute version node1 set 8.4.1
+noti 180 node1/crm: start ha group migration...
+noti 180 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 180 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 180 node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 180 node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 180 node1/crm: ha groups migration failed
+noti 180 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 220 cmdlist: execute reboot node1
+info 220 node1/lrm: got shutdown request with shutdown policy 'conditional'
+info 220 node1/lrm: reboot LRM, stop and freeze all services
+info 220 node1/crm: service 'vm:101': state changed from 'started' to 'freeze'
+info 221 node1/lrm: stopping service vm:101
+info 221 node1/lrm: service status vm:101 stopped
+info 222 node1/lrm: exit (loop end)
+info 222 reboot: execute crm node1 stop
+info 221 node1/crm: server received shutdown request
+info 240 node1/crm: voluntary release CRM lock
+info 241 node1/crm: exit (loop end)
+info 241 reboot: execute power node1 off
+info 241 reboot: execute power node1 on
+info 241 node1/crm: status change startup => wait_for_quorum
+info 240 node1/lrm: status change startup => wait_for_agent_lock
+info 242 node2/crm: got lock 'ha_manager_lock'
+info 242 node2/crm: status change slave => master
+info 242 node2/crm: service 'vm:101': state changed from 'freeze' to 'started'
+info 260 node1/crm: status change wait_for_quorum => slave
+info 261 node1/lrm: got lock 'ha_agent_node1_lock'
+info 261 node1/lrm: status change wait_for_agent_lock => active
+info 261 node1/lrm: starting service vm:101
+info 261 node1/lrm: service status vm:101 started
+noti 282 node2/crm: start ha group migration...
+noti 282 node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti 282 node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 282 node2/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 282 node2/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 282 node2/crm: ha groups migration failed
+noti 282 node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 320 cmdlist: execute version node2 set 8.4.1
+noti 402 node2/crm: start ha group migration...
+noti 402 node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti 402 node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 402 node2/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 402 node2/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 402 node2/crm: ha groups migration failed
+noti 402 node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 420 cmdlist: execute reboot node2
+info 420 node2/lrm: got shutdown request with shutdown policy 'conditional'
+info 420 node2/lrm: reboot LRM, stop and freeze all services
+info 422 node2/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info 423 node2/lrm: stopping service vm:102
+info 423 node2/lrm: service status vm:102 stopped
+info 424 node2/lrm: exit (loop end)
+info 424 reboot: execute crm node2 stop
+info 423 node2/crm: server received shutdown request
+info 442 node2/crm: voluntary release CRM lock
+info 443 node2/crm: exit (loop end)
+info 443 reboot: execute power node2 off
+info 443 reboot: execute power node2 on
+info 443 node2/crm: status change startup => wait_for_quorum
+info 440 node2/lrm: status change startup => wait_for_agent_lock
+info 444 node3/crm: got lock 'ha_manager_lock'
+info 444 node3/crm: status change slave => master
+info 444 node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info 462 node2/crm: status change wait_for_quorum => slave
+info 463 node2/lrm: got lock 'ha_agent_node2_lock'
+info 463 node2/lrm: status change wait_for_agent_lock => active
+info 463 node2/lrm: starting service vm:102
+info 463 node2/lrm: service status vm:102 started
+noti 484 node3/crm: start ha group migration...
+noti 484 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 484 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 484 node3/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 484 node3/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 484 node3/crm: ha groups migration failed
+noti 484 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 520 cmdlist: execute version node3 set 8.4.1
+noti 604 node3/crm: start ha group migration...
+noti 604 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 604 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 604 node3/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 604 node3/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 604 node3/crm: ha groups migration failed
+noti 604 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 620 cmdlist: execute reboot node3
+info 620 node3/lrm: got shutdown request with shutdown policy 'conditional'
+info 620 node3/lrm: reboot LRM, stop and freeze all services
+info 624 node3/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info 625 node3/lrm: stopping service vm:103
+info 625 node3/lrm: service status vm:103 stopped
+info 626 node3/lrm: exit (loop end)
+info 626 reboot: execute crm node3 stop
+info 625 node3/crm: server received shutdown request
+info 644 node3/crm: voluntary release CRM lock
+info 645 node3/crm: exit (loop end)
+info 645 reboot: execute power node3 off
+info 645 reboot: execute power node3 on
+info 645 node3/crm: status change startup => wait_for_quorum
+info 640 node3/lrm: status change startup => wait_for_agent_lock
+info 660 node1/crm: got lock 'ha_manager_lock'
+info 660 node1/crm: status change slave => master
+info 660 node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info 664 node3/crm: status change wait_for_quorum => slave
+info 665 node3/lrm: got lock 'ha_agent_node3_lock'
+info 665 node3/lrm: status change wait_for_agent_lock => active
+info 665 node3/lrm: starting service vm:103
+info 665 node3/lrm: service status vm:103 started
+noti 700 node1/crm: start ha group migration...
+noti 700 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 700 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 700 node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 700 node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 700 node1/crm: ha groups migration failed
+noti 700 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 720 cmdlist: execute version node1 set 9.0.0~16
+info 820 cmdlist: execute reboot node1
+info 820 node1/lrm: got shutdown request with shutdown policy 'conditional'
+info 820 node1/lrm: reboot LRM, stop and freeze all services
+noti 820 node1/crm: start ha group migration...
+noti 820 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 820 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'restart'
+err 820 node1/crm: abort ha groups migration: lrm 'node1' is not in mode 'active'
+err 820 node1/crm: ha groups migration failed
+noti 820 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 820 node1/crm: service 'vm:101': state changed from 'started' to 'freeze'
+info 821 node1/lrm: stopping service vm:101
+info 821 node1/lrm: service status vm:101 stopped
+info 822 node1/lrm: exit (loop end)
+info 822 reboot: execute crm node1 stop
+info 821 node1/crm: server received shutdown request
+info 840 node1/crm: voluntary release CRM lock
+info 841 node1/crm: exit (loop end)
+info 841 reboot: execute power node1 off
+info 841 reboot: execute power node1 on
+info 841 node1/crm: status change startup => wait_for_quorum
+info 840 node1/lrm: status change startup => wait_for_agent_lock
+info 842 node2/crm: got lock 'ha_manager_lock'
+info 842 node2/crm: status change slave => master
+info 842 node2/crm: service 'vm:101': state changed from 'freeze' to 'started'
+info 860 node1/crm: status change wait_for_quorum => slave
+info 861 node1/lrm: got lock 'ha_agent_node1_lock'
+info 861 node1/lrm: status change wait_for_agent_lock => active
+info 861 node1/lrm: starting service vm:101
+info 861 node1/lrm: service status vm:101 started
+noti 882 node2/crm: start ha group migration...
+noti 882 node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti 882 node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 882 node2/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 882 node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti 882 node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 882 node2/crm: ha groups migration: node 'node2' has version '8.4.1'
+err 882 node2/crm: abort ha groups migration: node 'node2' needs at least pve-manager version '9.0.0~16'
+err 882 node2/crm: ha groups migration failed
+noti 882 node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 920 cmdlist: execute version node2 set 9.0.0~16
+noti 1002 node2/crm: start ha group migration...
+noti 1002 node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1002 node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1002 node2/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 1002 node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1002 node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1002 node2/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti 1002 node2/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1002 node2/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1002 node2/crm: ha groups migration: node 'node3' has version '8.4.1'
+err 1002 node2/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 1002 node2/crm: ha groups migration failed
+noti 1002 node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1020 cmdlist: execute reboot node2
+info 1020 node2/lrm: got shutdown request with shutdown policy 'conditional'
+info 1020 node2/lrm: reboot LRM, stop and freeze all services
+info 1022 node2/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info 1023 node2/lrm: stopping service vm:102
+info 1023 node2/lrm: service status vm:102 stopped
+info 1024 node2/lrm: exit (loop end)
+info 1024 reboot: execute crm node2 stop
+info 1023 node2/crm: server received shutdown request
+info 1042 node2/crm: voluntary release CRM lock
+info 1043 node2/crm: exit (loop end)
+info 1043 reboot: execute power node2 off
+info 1043 reboot: execute power node2 on
+info 1043 node2/crm: status change startup => wait_for_quorum
+info 1040 node2/lrm: status change startup => wait_for_agent_lock
+info 1044 node3/crm: got lock 'ha_manager_lock'
+info 1044 node3/crm: status change slave => master
+info 1044 node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info 1062 node2/crm: status change wait_for_quorum => slave
+info 1063 node2/lrm: got lock 'ha_agent_node2_lock'
+info 1063 node2/lrm: status change wait_for_agent_lock => active
+info 1063 node2/lrm: starting service vm:102
+info 1063 node2/lrm: service status vm:102 started
+noti 1084 node3/crm: start ha group migration...
+noti 1084 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1084 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1084 node3/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 1084 node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1084 node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1084 node3/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti 1084 node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1084 node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1084 node3/crm: ha groups migration: node 'node3' has version '8.4.1'
+err 1084 node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 1084 node3/crm: ha groups migration failed
+noti 1084 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1120 cmdlist: execute version node3 set 9.0.0~15
+noti 1204 node3/crm: start ha group migration...
+noti 1204 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1204 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1204 node3/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 1204 node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1204 node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1204 node3/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti 1204 node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1204 node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1204 node3/crm: ha groups migration: node 'node3' has version '9.0.0~15'
+err 1204 node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 1204 node3/crm: ha groups migration failed
+noti 1204 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1220 cmdlist: execute reboot node3
+info 1220 node3/lrm: got shutdown request with shutdown policy 'conditional'
+info 1220 node3/lrm: reboot LRM, stop and freeze all services
+info 1224 node3/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info 1225 node3/lrm: stopping service vm:103
+info 1225 node3/lrm: service status vm:103 stopped
+info 1226 node3/lrm: exit (loop end)
+info 1226 reboot: execute crm node3 stop
+info 1225 node3/crm: server received shutdown request
+info 1244 node3/crm: voluntary release CRM lock
+info 1245 node3/crm: exit (loop end)
+info 1245 reboot: execute power node3 off
+info 1245 reboot: execute power node3 on
+info 1245 node3/crm: status change startup => wait_for_quorum
+info 1240 node3/lrm: status change startup => wait_for_agent_lock
+info 1260 node1/crm: got lock 'ha_manager_lock'
+info 1260 node1/crm: status change slave => master
+info 1260 node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info 1264 node3/crm: status change wait_for_quorum => slave
+info 1265 node3/lrm: got lock 'ha_agent_node3_lock'
+info 1265 node3/lrm: status change wait_for_agent_lock => active
+info 1265 node3/lrm: starting service vm:103
+info 1265 node3/lrm: service status vm:103 started
+noti 1300 node1/crm: start ha group migration...
+noti 1300 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1300 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1300 node1/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 1300 node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1300 node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1300 node1/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti 1300 node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1300 node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1300 node1/crm: ha groups migration: node 'node3' has version '9.0.0~15'
+err 1300 node1/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 1300 node1/crm: ha groups migration failed
+noti 1300 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1320 cmdlist: execute version node3 set 9.0.0~17
+info 1420 cmdlist: execute reboot node3
+info 1420 node3/lrm: got shutdown request with shutdown policy 'conditional'
+info 1420 node3/lrm: reboot LRM, stop and freeze all services
+noti 1420 node1/crm: start ha group migration...
+noti 1420 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1420 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1420 node1/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 1420 node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1420 node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1420 node1/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti 1420 node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1420 node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'restart'
+err 1420 node1/crm: abort ha groups migration: lrm 'node3' is not in mode 'active'
+err 1420 node1/crm: ha groups migration failed
+noti 1420 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1420 node1/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info 1425 node3/lrm: stopping service vm:103
+info 1425 node3/lrm: service status vm:103 stopped
+info 1426 node3/lrm: exit (loop end)
+info 1426 reboot: execute crm node3 stop
+info 1425 node3/crm: server received shutdown request
+info 1445 node3/crm: exit (loop end)
+info 1445 reboot: execute power node3 off
+info 1445 reboot: execute power node3 on
+info 1445 node3/crm: status change startup => wait_for_quorum
+info 1440 node3/lrm: status change startup => wait_for_agent_lock
+info 1460 node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info 1464 node3/crm: status change wait_for_quorum => slave
+info 1465 node3/lrm: got lock 'ha_agent_node3_lock'
+info 1465 node3/lrm: status change wait_for_agent_lock => active
+info 1465 node3/lrm: starting service vm:103
+info 1465 node3/lrm: service status vm:103 started
+noti 1540 node1/crm: start ha group migration...
+noti 1540 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1540 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1540 node1/crm: ha groups migration: node 'node1' has version '9.0.0~16'
+noti 1540 node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1540 node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1540 node1/crm: ha groups migration: node 'node2' has version '9.0.0~16'
+noti 1540 node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1540 node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1540 node1/crm: ha groups migration: node 'node3' has version '9.0.0~17'
+noti 1540 node1/crm: ha groups migration: migration to rules config successful
+noti 1540 node1/crm: ha groups migration: migration to resources config successful
+noti 1540 node1/crm: ha groups migration: group config deletion successful
+noti 1540 node1/crm: ha groups migration successful
+info 2020 hardware: exit simulation - done
diff --git a/src/test/test-group-migrate3/manager_status b/src/test/test-group-migrate3/manager_status
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/test/test-group-migrate3/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-group-migrate3/service_config b/src/test/test-group-migrate3/service_config
new file mode 100644
index 00000000..a27551e5
--- /dev/null
+++ b/src/test/test-group-migrate3/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "started", "group": "group1" },
+ "vm:102": { "node": "node2", "state": "started", "group": "group2" },
+ "vm:103": { "node": "node3", "state": "started", "group": "group2" }
+}
diff --git a/src/test/test-group-migrate4/README b/src/test/test-group-migrate4/README
new file mode 100644
index 00000000..37e60c7d
--- /dev/null
+++ b/src/test/test-group-migrate4/README
@@ -0,0 +1,8 @@
+Test whether a cluster, where all nodes have the same version from the previous
+major release, can be properly upgraded to the neede major release version and
+then the CRM correctly migrates the HA group config only after all nodes have
+the minimum major release version.
+
+Additionally, the nodes are rebooted with every version upgrade and in-between
+the CFS sporadically fails to read/write, fails to update cluster state and an
+LRM is restarted, which all prohibit the HA groups config migration.
diff --git a/src/test/test-group-migrate4/cmdlist b/src/test/test-group-migrate4/cmdlist
new file mode 100644
index 00000000..fdd3bfdd
--- /dev/null
+++ b/src/test/test-group-migrate4/cmdlist
@@ -0,0 +1,15 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on" ],
+ [ "delay 10" ],
+ [ "version node1 set 9.0.0" ],
+ [ "reboot node1" ],
+ [ "cfs node2 rw fail" ],
+ [ "version node2 set 9.0.0" ],
+ [ "cfs node2 rw work" ],
+ [ "reboot node2" ],
+ [ "cfs node3 update fail" ],
+ [ "cfs node3 update work" ],
+ [ "version node3 set 9.0.1" ],
+ [ "restart-lrm node2" ],
+ [ "reboot node3" ]
+]
diff --git a/src/test/test-group-migrate4/groups b/src/test/test-group-migrate4/groups
new file mode 100644
index 00000000..bad746ca
--- /dev/null
+++ b/src/test/test-group-migrate4/groups
@@ -0,0 +1,7 @@
+group: group1
+ nodes node1
+ restricted 1
+
+group: group2
+ nodes node2:2,node3
+ nofailback 1
diff --git a/src/test/test-group-migrate4/hardware_status b/src/test/test-group-migrate4/hardware_status
new file mode 100644
index 00000000..7ad46416
--- /dev/null
+++ b/src/test/test-group-migrate4/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "version": "8.4.1" },
+ "node2": { "power": "off", "network": "off", "version": "8.4.1" },
+ "node3": { "power": "off", "network": "off", "version": "8.4.1" }
+}
diff --git a/src/test/test-group-migrate4/log.expect b/src/test/test-group-migrate4/log.expect
new file mode 100644
index 00000000..7ffe33e3
--- /dev/null
+++ b/src/test/test-group-migrate4/log.expect
@@ -0,0 +1,277 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:101': state changed from 'request_start' to 'started' (node = node1)
+info 20 node1/crm: service 'vm:102': state changed from 'request_start' to 'started' (node = node2)
+info 20 node1/crm: service 'vm:103': state changed from 'request_start' to 'started' (node = node3)
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+noti 60 node1/crm: start ha group migration...
+noti 60 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 60 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 60 node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 60 node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 60 node1/crm: ha groups migration failed
+noti 60 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 120 cmdlist: execute delay 10
+noti 180 node1/crm: start ha group migration...
+noti 180 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 180 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 180 node1/crm: ha groups migration: node 'node1' has version '8.4.1'
+err 180 node1/crm: abort ha groups migration: node 'node1' needs at least pve-manager version '9.0.0~16'
+err 180 node1/crm: ha groups migration failed
+noti 180 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 220 cmdlist: execute version node1 set 9.0.0
+noti 300 node1/crm: start ha group migration...
+noti 300 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 300 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 300 node1/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 300 node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti 300 node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 300 node1/crm: ha groups migration: node 'node2' has version '8.4.1'
+err 300 node1/crm: abort ha groups migration: node 'node2' needs at least pve-manager version '9.0.0~16'
+err 300 node1/crm: ha groups migration failed
+noti 300 node1/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 320 cmdlist: execute reboot node1
+info 320 node1/lrm: got shutdown request with shutdown policy 'conditional'
+info 320 node1/lrm: reboot LRM, stop and freeze all services
+info 320 node1/crm: service 'vm:101': state changed from 'started' to 'freeze'
+info 321 node1/lrm: stopping service vm:101
+info 321 node1/lrm: service status vm:101 stopped
+info 322 node1/lrm: exit (loop end)
+info 322 reboot: execute crm node1 stop
+info 321 node1/crm: server received shutdown request
+info 340 node1/crm: voluntary release CRM lock
+info 341 node1/crm: exit (loop end)
+info 341 reboot: execute power node1 off
+info 341 reboot: execute power node1 on
+info 341 node1/crm: status change startup => wait_for_quorum
+info 340 node1/lrm: status change startup => wait_for_agent_lock
+info 342 node2/crm: got lock 'ha_manager_lock'
+info 342 node2/crm: status change slave => master
+info 342 node2/crm: service 'vm:101': state changed from 'freeze' to 'started'
+info 360 node1/crm: status change wait_for_quorum => slave
+info 361 node1/lrm: got lock 'ha_agent_node1_lock'
+info 361 node1/lrm: status change wait_for_agent_lock => active
+info 361 node1/lrm: starting service vm:101
+info 361 node1/lrm: service status vm:101 started
+noti 382 node2/crm: start ha group migration...
+noti 382 node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti 382 node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 382 node2/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 382 node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti 382 node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 382 node2/crm: ha groups migration: node 'node2' has version '8.4.1'
+err 382 node2/crm: abort ha groups migration: node 'node2' needs at least pve-manager version '9.0.0~16'
+err 382 node2/crm: ha groups migration failed
+noti 382 node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 420 cmdlist: execute cfs node2 rw fail
+err 422 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 422 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 423 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 423 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 423 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 442 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 442 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 443 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 443 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 443 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 462 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 462 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 463 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 463 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 463 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 482 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 482 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 483 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 483 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 483 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 502 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 502 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 503 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 503 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 503 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+info 520 cmdlist: execute version node2 set 9.0.0
+err 522 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 522 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 523 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 523 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 523 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 542 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 542 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 543 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 543 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 543 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 562 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 562 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 563 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 563 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 563 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 582 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 582 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 583 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 583 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 583 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+err 602 node2/crm: could not read manager status: cfs connection refused - not mounted?
+err 602 node2/crm: got unexpected error - cfs connection refused - not mounted?
+err 603 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 603 node2/lrm: updating service status from manager failed: cfs connection refused - not mounted?
+err 603 node2/lrm: unable to write lrm status file - cfs connection refused - not mounted?
+info 620 cmdlist: execute cfs node2 rw work
+noti 702 node2/crm: start ha group migration...
+noti 702 node2/crm: ha groups migration: node 'node1' is in state 'online'
+noti 702 node2/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 702 node2/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 702 node2/crm: ha groups migration: node 'node2' is in state 'online'
+noti 702 node2/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 702 node2/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti 702 node2/crm: ha groups migration: node 'node3' is in state 'online'
+noti 702 node2/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 702 node2/crm: ha groups migration: node 'node3' has version '8.4.1'
+err 702 node2/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 702 node2/crm: ha groups migration failed
+noti 702 node2/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 720 cmdlist: execute reboot node2
+info 720 node2/lrm: got shutdown request with shutdown policy 'conditional'
+info 720 node2/lrm: reboot LRM, stop and freeze all services
+info 722 node2/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info 723 node2/lrm: stopping service vm:102
+info 723 node2/lrm: service status vm:102 stopped
+info 724 node2/lrm: exit (loop end)
+info 724 reboot: execute crm node2 stop
+info 723 node2/crm: server received shutdown request
+info 742 node2/crm: voluntary release CRM lock
+info 743 node2/crm: exit (loop end)
+info 743 reboot: execute power node2 off
+info 743 reboot: execute power node2 on
+info 743 node2/crm: status change startup => wait_for_quorum
+info 740 node2/lrm: status change startup => wait_for_agent_lock
+info 744 node3/crm: got lock 'ha_manager_lock'
+info 744 node3/crm: status change slave => master
+info 744 node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info 762 node2/crm: status change wait_for_quorum => slave
+info 763 node2/lrm: got lock 'ha_agent_node2_lock'
+info 763 node2/lrm: status change wait_for_agent_lock => active
+info 763 node2/lrm: starting service vm:102
+info 763 node2/lrm: service status vm:102 started
+noti 784 node3/crm: start ha group migration...
+noti 784 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 784 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 784 node3/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 784 node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti 784 node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 784 node3/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti 784 node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti 784 node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 784 node3/crm: ha groups migration: node 'node3' has version '8.4.1'
+err 784 node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 784 node3/crm: ha groups migration failed
+noti 784 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 820 cmdlist: execute cfs node3 update fail
+noti 824 node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 825 node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 844 node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 845 node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 864 node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 865 node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 884 node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 885 node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 904 node3/crm: temporary inconsistent cluster state (cfs restart?), skip round
+noti 905 node3/lrm: temporary inconsistent cluster state (cfs restart?), skip round
+info 920 cmdlist: execute cfs node3 update work
+noti 1004 node3/crm: start ha group migration...
+noti 1004 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1004 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1004 node3/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 1004 node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1004 node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1004 node3/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti 1004 node3/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1004 node3/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1004 node3/crm: ha groups migration: node 'node3' has version '8.4.1'
+err 1004 node3/crm: abort ha groups migration: node 'node3' needs at least pve-manager version '9.0.0~16'
+err 1004 node3/crm: ha groups migration failed
+noti 1004 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1020 cmdlist: execute version node3 set 9.0.1
+info 1120 cmdlist: execute restart-lrm node2
+info 1120 node2/lrm: restart LRM, freeze all services
+noti 1124 node3/crm: start ha group migration...
+noti 1124 node3/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1124 node3/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1124 node3/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 1124 node3/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1124 node3/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'restart'
+err 1124 node3/crm: abort ha groups migration: lrm 'node2' is not in mode 'active'
+err 1124 node3/crm: ha groups migration failed
+noti 1124 node3/crm: retry ha groups migration in 6 rounds (~ 60 seconds)
+info 1124 node3/crm: service 'vm:102': state changed from 'started' to 'freeze'
+info 1144 node2/lrm: exit (loop end)
+info 1144 node2/lrm: status change startup => wait_for_agent_lock
+info 1164 node3/crm: service 'vm:102': state changed from 'freeze' to 'started'
+info 1183 node2/lrm: got lock 'ha_agent_node2_lock'
+info 1183 node2/lrm: status change wait_for_agent_lock => active
+info 1220 cmdlist: execute reboot node3
+info 1220 node3/lrm: got shutdown request with shutdown policy 'conditional'
+info 1220 node3/lrm: reboot LRM, stop and freeze all services
+info 1224 node3/crm: service 'vm:103': state changed from 'started' to 'freeze'
+info 1225 node3/lrm: stopping service vm:103
+info 1225 node3/lrm: service status vm:103 stopped
+info 1226 node3/lrm: exit (loop end)
+info 1226 reboot: execute crm node3 stop
+info 1225 node3/crm: server received shutdown request
+info 1244 node3/crm: voluntary release CRM lock
+info 1245 node3/crm: exit (loop end)
+info 1245 reboot: execute power node3 off
+info 1245 reboot: execute power node3 on
+info 1245 node3/crm: status change startup => wait_for_quorum
+info 1240 node3/lrm: status change startup => wait_for_agent_lock
+info 1260 node1/crm: got lock 'ha_manager_lock'
+info 1260 node1/crm: status change slave => master
+info 1260 node1/crm: service 'vm:103': state changed from 'freeze' to 'started'
+info 1264 node3/crm: status change wait_for_quorum => slave
+info 1265 node3/lrm: got lock 'ha_agent_node3_lock'
+info 1265 node3/lrm: status change wait_for_agent_lock => active
+info 1265 node3/lrm: starting service vm:103
+info 1265 node3/lrm: service status vm:103 started
+noti 1300 node1/crm: start ha group migration...
+noti 1300 node1/crm: ha groups migration: node 'node1' is in state 'online'
+noti 1300 node1/crm: ha groups migration: lrm 'node1' is in state 'active' and mode 'active'
+noti 1300 node1/crm: ha groups migration: node 'node1' has version '9.0.0'
+noti 1300 node1/crm: ha groups migration: node 'node2' is in state 'online'
+noti 1300 node1/crm: ha groups migration: lrm 'node2' is in state 'active' and mode 'active'
+noti 1300 node1/crm: ha groups migration: node 'node2' has version '9.0.0'
+noti 1300 node1/crm: ha groups migration: node 'node3' is in state 'online'
+noti 1300 node1/crm: ha groups migration: lrm 'node3' is in state 'active' and mode 'active'
+noti 1300 node1/crm: ha groups migration: node 'node3' has version '9.0.1'
+noti 1300 node1/crm: ha groups migration: migration to rules config successful
+noti 1300 node1/crm: ha groups migration: migration to resources config successful
+noti 1300 node1/crm: ha groups migration: group config deletion successful
+noti 1300 node1/crm: ha groups migration successful
+info 1820 hardware: exit simulation - done
diff --git a/src/test/test-group-migrate4/manager_status b/src/test/test-group-migrate4/manager_status
new file mode 100644
index 00000000..9e26dfee
--- /dev/null
+++ b/src/test/test-group-migrate4/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-group-migrate4/service_config b/src/test/test-group-migrate4/service_config
new file mode 100644
index 00000000..a27551e5
--- /dev/null
+++ b/src/test/test-group-migrate4/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "started", "group": "group1" },
+ "vm:102": { "node": "node2", "state": "started", "group": "group2" },
+ "vm:103": { "node": "node3", "state": "started", "group": "group2" }
+}
--
2.47.2
More information about the pve-devel
mailing list