[pve-devel] [RFC ha-manager v3 15/15] manager: persistently migrate ha groups to ha rules
Daniel Kral
d.kral at proxmox.com
Fri Jul 4 20:16:55 CEST 2025
Migrate the HA groups config to the HA resources and HA rules config
persistently on disk and retry until it succeeds.
The HA group config is already migrated in the HA Manager in-memory, but
to persistently use them as HA node affinity rules, they must be
migrated to the HA rules config.
As the new 'failback' flag can only be read by newer HA Manager versions
and the rules config cannot be read by older HA Manager versions, these
can only be migrated (for the HA resources config) and deleted (for the
HA groups config) if all nodes are upgraded to the correct pve-manager
version, which has a version dependency on the ha-manager package, which
can read and apply the HA rules.
Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
As already mentioned in the cover letter, this is only a rough draft how
it should work, but there are at least two things missing (and I'm
unfortunately sure that there will be more edge cases than this happy
path), but I hope I got it right close enough to be a viable solution.
This also sparks the discussion on how we should handle any uses of the
HA group API now, as those haven't been adapted yet. We could also
remove them here, or make them (+ the HA resource API 'group' property)
translate the HA groups to HA rules every time..
src/PVE/HA/Config.pm | 5 +
src/PVE/HA/Env.pm | 24 ++
src/PVE/HA/Env/PVE2.pm | 28 ++
src/PVE/HA/Manager.pm | 94 ++++++
src/PVE/HA/Sim/Env.pm | 30 ++
src/PVE/HA/Sim/Hardware.pm | 22 ++
src/test/test-group-migrate1/README | 10 +
src/test/test-group-migrate1/cmdlist | 3 +
src/test/test-group-migrate1/groups | 7 +
src/test/test-group-migrate1/hardware_status | 5 +
src/test/test-group-migrate1/log.expect | 306 +++++++++++++++++++
src/test/test-group-migrate1/manager_status | 1 +
src/test/test-group-migrate1/service_config | 5 +
src/test/test-group-migrate2/README | 10 +
src/test/test-group-migrate2/cmdlist | 3 +
src/test/test-group-migrate2/groups | 7 +
src/test/test-group-migrate2/hardware_status | 5 +
src/test/test-group-migrate2/log.expect | 47 +++
src/test/test-group-migrate2/manager_status | 1 +
src/test/test-group-migrate2/service_config | 5 +
20 files changed, 618 insertions(+)
create mode 100644 src/test/test-group-migrate1/README
create mode 100644 src/test/test-group-migrate1/cmdlist
create mode 100644 src/test/test-group-migrate1/groups
create mode 100644 src/test/test-group-migrate1/hardware_status
create mode 100644 src/test/test-group-migrate1/log.expect
create mode 100644 src/test/test-group-migrate1/manager_status
create mode 100644 src/test/test-group-migrate1/service_config
create mode 100644 src/test/test-group-migrate2/README
create mode 100644 src/test/test-group-migrate2/cmdlist
create mode 100644 src/test/test-group-migrate2/groups
create mode 100644 src/test/test-group-migrate2/hardware_status
create mode 100644 src/test/test-group-migrate2/log.expect
create mode 100644 src/test/test-group-migrate2/manager_status
create mode 100644 src/test/test-group-migrate2/service_config
diff --git a/src/PVE/HA/Config.pm b/src/PVE/HA/Config.pm
index 424a6e1..59bafd7 100644
--- a/src/PVE/HA/Config.pm
+++ b/src/PVE/HA/Config.pm
@@ -234,6 +234,11 @@ sub read_group_config {
return cfs_read_file($ha_groups_config);
}
+sub delete_group_config {
+
+ unlink $ha_groups_config or die "failed to remove group config: $!\n";
+}
+
sub write_group_config {
my ($cfg) = @_;
diff --git a/src/PVE/HA/Env.pm b/src/PVE/HA/Env.pm
index 5cee7b3..1325676 100644
--- a/src/PVE/HA/Env.pm
+++ b/src/PVE/HA/Env.pm
@@ -100,6 +100,12 @@ sub update_service_config {
return $self->{plug}->update_service_config($sid, $param);
}
+sub write_service_config {
+ my ($self, $conf) = @_;
+
+ $self->{plug}->write_service_config($conf);
+}
+
sub parse_sid {
my ($self, $sid) = @_;
@@ -137,12 +143,24 @@ sub read_rules_config {
return $self->{plug}->read_rules_config();
}
+sub write_rules_config {
+ my ($self, $rules) = @_;
+
+ $self->{plug}->write_rules_config($rules);
+}
+
sub read_group_config {
my ($self) = @_;
return $self->{plug}->read_group_config();
}
+sub delete_group_config {
+ my ($self) = @_;
+
+ $self->{plug}->delete_group_config();
+}
+
# this should return a hash containing info
# what nodes are members and online.
sub get_node_info {
@@ -288,4 +306,10 @@ sub get_static_node_stats {
return $self->{plug}->get_static_node_stats();
}
+sub get_node_version {
+ my ($self, $node) = @_;
+
+ return $self->{plug}->get_node_version($node);
+}
+
1;
diff --git a/src/PVE/HA/Env/PVE2.pm b/src/PVE/HA/Env/PVE2.pm
index 58fd36e..aecffc0 100644
--- a/src/PVE/HA/Env/PVE2.pm
+++ b/src/PVE/HA/Env/PVE2.pm
@@ -141,6 +141,12 @@ sub update_service_config {
return PVE::HA::Config::update_resources_config($sid, $param);
}
+sub write_service_config {
+ my ($self, $conf) = @_;
+
+ return PVE::HA::Config::write_resources_config($conf);
+}
+
sub parse_sid {
my ($self, $sid) = @_;
@@ -201,12 +207,24 @@ sub read_rules_config {
return PVE::HA::Config::read_and_check_rules_config();
}
+sub write_rules_config {
+ my ($self, $rules) = @_;
+
+ PVE::HA::Config::write_rules_config($rules);
+}
+
sub read_group_config {
my ($self) = @_;
return PVE::HA::Config::read_group_config();
}
+sub delete_group_config {
+ my ($self) = @_;
+
+ PVE::HA::Config::delete_group_config();
+}
+
# this should return a hash containing info
# what nodes are members and online.
sub get_node_info {
@@ -489,4 +507,14 @@ sub get_static_node_stats {
return $stats;
}
+sub get_node_version {
+ my ($self, $node) = @_;
+
+ my $version_info = PVE::Cluster::get_node_kv('version-info', $node);
+
+ return undef if !$version_info->{$node};
+
+ return $version_info->{$node}->{version};
+}
+
1;
diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 4357253..b2fd896 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -464,6 +464,97 @@ sub update_crm_commands {
}
+# TODO Call needs to be abstracted to environment as test environment has 3 default
+# groups, and will then evalulate true for each test
+sub have_groups_been_migrated {
+ my ($haenv) = @_;
+
+ my $groups = $haenv->read_group_config();
+
+ return 1 if !$groups;
+ return keys $groups->{ids}->%* < 1;
+}
+
+my $get_version_parts = sub {
+ my ($node_version) = @_;
+
+ my ($maj, $min) = $node_version =~ m/^(\d+)\.(\d+)/;
+
+ return ($maj, $min);
+};
+
+my $has_node_min_version = sub {
+ my ($node_version, $min_version) = @_;
+
+ my ($min_major, $min_minor) = $get_version_parts->($min_version);
+ my ($maj, $min) = $get_version_parts->($node_version);
+
+ return 0 if $maj < $min_major;
+ return 0 if $maj == $min_major && $min < $min_minor;
+
+ return 1;
+};
+
+# TODO PVE 10: Remove group migration when HA groups have been fully migrated to rules
+sub migrate_groups_persistently {
+ my ($haenv, $ns) = @_;
+
+ return 1 if have_groups_been_migrated($haenv); # groups already migrated
+
+ $haenv->log('notice', "Start migrating HA groups...");
+
+ # NOTE pve-manager has a version dependency on the ha-manager which supports HA rules
+ my $HA_RULES_MINVERSION = "9.0.0~2";
+
+ eval {
+ my $resources = $haenv->read_service_config();
+ my $groups = $haenv->read_group_config();
+ my $rules = $haenv->read_rules_config();
+
+ # write changes to rules config whenever possible to allow users to
+ # already modify migrated rules
+ PVE::HA::Groups::migrate_groups_to_rules($rules, $groups, $resources);
+ $haenv->write_rules_config($rules);
+ $haenv->log('notice', "HA groups to rules config migration successful");
+
+ for my $node ($ns->list_nodes()->@*) {
+ my $node_status = $ns->get_node_state($node);
+ $haenv->log(
+ 'notice',
+ "node '$node' is in state '$node_status' during HA group migration.",
+ );
+ die "node '$node' is not online\n" if $node_status ne 'online';
+
+ my $node_version = $haenv->get_node_version($node);
+ die "could not retrieve version from node '$node'\n" if !$node_version;
+ $haenv->log('notice', "Node '$node' has pve-manager version '$node_version'");
+
+ my $has_min_version = $has_node_min_version->($node_version, $HA_RULES_MINVERSION);
+
+ die "node '$node' needs at least '$HA_RULES_MINVERSION' to migrate HA groups\n"
+ if !$has_min_version;
+ }
+
+ # write changes to resources config only after node checks, because old
+ # nodes cannot read the 'failback' flag yet
+ PVE::HA::Groups::migrate_groups_to_resources($groups, $resources);
+ $haenv->write_service_config($resources);
+ $haenv->log('notice', "HA groups to services config migration successful");
+
+ $haenv->delete_group_config();
+
+ $haenv->log('notice', "HA groups config deletion successful");
+ };
+ if (my $err = $@) {
+ $haenv->log('err', "Abort HA group migration: $err");
+ return 0;
+ }
+
+ $haenv->log('notice', "HA groups migration successful");
+
+ return 1;
+}
+
sub manage {
my ($self) = @_;
@@ -481,6 +572,9 @@ sub manage {
$self->update_crs_scheduler_mode();
+ # TODO Should only be run every couple of manager rounds
+ migrate_groups_persistently($haenv, $ns) if !have_groups_been_migrated($haenv);
+
my ($sc, $services_digest) = $haenv->read_service_config();
$self->{groups} = $haenv->read_group_config(); # update
diff --git a/src/PVE/HA/Sim/Env.pm b/src/PVE/HA/Sim/Env.pm
index bb76b7f..446071d 100644
--- a/src/PVE/HA/Sim/Env.pm
+++ b/src/PVE/HA/Sim/Env.pm
@@ -215,6 +215,14 @@ sub update_service_config {
return $self->{hardware}->update_service_config($sid, $param);
}
+sub write_service_config {
+ my ($self, $conf) = @_;
+
+ $assert_cfs_can_rw->($self);
+
+ $self->{hardware}->write_service_config($conf);
+}
+
sub parse_sid {
my ($self, $sid) = @_;
@@ -259,6 +267,14 @@ sub read_rules_config {
return $self->{hardware}->read_rules_config();
}
+sub write_rules_config {
+ my ($self, $rules) = @_;
+
+ $assert_cfs_can_rw->($self);
+
+ $self->{hardware}->write_rules_config($rules);
+}
+
sub read_group_config {
my ($self) = @_;
@@ -267,6 +283,14 @@ sub read_group_config {
return $self->{hardware}->read_group_config();
}
+sub delete_group_config {
+ my ($self) = @_;
+
+ $assert_cfs_can_rw->($self);
+
+ $self->{hardware}->delete_group_config();
+}
+
# this is normally only allowed by the master to recover a _fenced_ service
sub steal_service {
my ($self, $sid, $current_node, $new_node) = @_;
@@ -468,4 +492,10 @@ sub get_static_node_stats {
return $self->{hardware}->get_static_node_stats();
}
+sub get_node_version {
+ my ($self, $node) = @_;
+
+ return $self->{hardware}->get_node_version($node);
+}
+
1;
diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index 579be2a..a44bc9b 100644
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -337,6 +337,13 @@ sub read_rules_config {
return $rules;
}
+sub write_rules_config {
+ my ($self, $rules) = @_;
+
+ my $filename = "$self->{statusdir}/rules_config";
+ PVE::HA::Rules->write_config($filename, $rules);
+}
+
sub read_group_config {
my ($self) = @_;
@@ -347,6 +354,13 @@ sub read_group_config {
return PVE::HA::Groups->parse_config($filename, $raw);
}
+sub delete_group_config {
+ my ($self) = @_;
+
+ my $filename = "$self->{statusdir}/groups";
+ unlink $filename or die "failed to remove group config: $!\n";
+}
+
sub read_service_status {
my ($self, $node) = @_;
@@ -942,4 +956,12 @@ sub get_static_node_stats {
return $stats;
}
+sub get_node_version {
+ my ($self, $node) = @_;
+
+ my $cstatus = $self->read_hardware_status_nolock();
+
+ return $cstatus->{$node}->{version} // "9.0.0~2";
+}
+
1;
diff --git a/src/test/test-group-migrate1/README b/src/test/test-group-migrate1/README
new file mode 100644
index 0000000..8775b6c
--- /dev/null
+++ b/src/test/test-group-migrate1/README
@@ -0,0 +1,10 @@
+Test whether a service in a unrestricted group will automatically migrate back
+to a node member in case of a manual migration to a non-member node.
+
+The test scenario is:
+- vm:101 should be kept on node3
+- vm:101 is currently running on node3
+
+The expected outcome is:
+- As vm:101 is manually migrated to node2, it is migrated back to node3, as
+ node3 is a group member and has higher priority than the other nodes
diff --git a/src/test/test-group-migrate1/cmdlist b/src/test/test-group-migrate1/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-group-migrate1/cmdlist
@@ -0,0 +1,3 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-group-migrate1/groups b/src/test/test-group-migrate1/groups
new file mode 100644
index 0000000..bad746c
--- /dev/null
+++ b/src/test/test-group-migrate1/groups
@@ -0,0 +1,7 @@
+group: group1
+ nodes node1
+ restricted 1
+
+group: group2
+ nodes node2:2,node3
+ nofailback 1
diff --git a/src/test/test-group-migrate1/hardware_status b/src/test/test-group-migrate1/hardware_status
new file mode 100644
index 0000000..d66fbf0
--- /dev/null
+++ b/src/test/test-group-migrate1/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "version": "9.0.0~2" },
+ "node2": { "power": "off", "network": "off", "version": "9.0.0~2" },
+ "node3": { "power": "off", "network": "off", "version": "8.4.1" }
+}
diff --git a/src/test/test-group-migrate1/log.expect b/src/test/test-group-migrate1/log.expect
new file mode 100644
index 0000000..d0ddf3d
--- /dev/null
+++ b/src/test/test-group-migrate1/log.expect
@@ -0,0 +1,306 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+noti 20 node1/crm: Start migrating HA groups...
+noti 20 node1/crm: HA groups to rules config migration successful
+noti 20 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 20 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 20 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 20 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 20 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 20 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 20 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:101': state changed from 'request_start' to 'started' (node = node1)
+info 20 node1/crm: service 'vm:102': state changed from 'request_start' to 'started' (node = node2)
+info 20 node1/crm: service 'vm:103': state changed from 'request_start' to 'started' (node = node3)
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+noti 40 node1/crm: Start migrating HA groups...
+noti 40 node1/crm: HA groups to rules config migration successful
+noti 40 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 40 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 40 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 40 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 40 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 40 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 40 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 60 node1/crm: Start migrating HA groups...
+noti 60 node1/crm: HA groups to rules config migration successful
+noti 60 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 60 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 60 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 60 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 60 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 60 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 60 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 80 node1/crm: Start migrating HA groups...
+noti 80 node1/crm: HA groups to rules config migration successful
+noti 80 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 80 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 80 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 80 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 80 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 80 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 80 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 100 node1/crm: Start migrating HA groups...
+noti 100 node1/crm: HA groups to rules config migration successful
+noti 100 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 100 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 100 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 100 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 100 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 100 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 100 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 120 node1/crm: Start migrating HA groups...
+noti 120 node1/crm: HA groups to rules config migration successful
+noti 120 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 120 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 120 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 120 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 120 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 120 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 120 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 140 node1/crm: Start migrating HA groups...
+noti 140 node1/crm: HA groups to rules config migration successful
+noti 140 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 140 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 140 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 140 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 140 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 140 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 140 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 160 node1/crm: Start migrating HA groups...
+noti 160 node1/crm: HA groups to rules config migration successful
+noti 160 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 160 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 160 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 160 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 160 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 160 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 160 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 180 node1/crm: Start migrating HA groups...
+noti 180 node1/crm: HA groups to rules config migration successful
+noti 180 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 180 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 180 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 180 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 180 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 180 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 180 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 200 node1/crm: Start migrating HA groups...
+noti 200 node1/crm: HA groups to rules config migration successful
+noti 200 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 200 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 200 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 200 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 200 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 200 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 200 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 220 node1/crm: Start migrating HA groups...
+noti 220 node1/crm: HA groups to rules config migration successful
+noti 220 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 220 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 220 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 220 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 220 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 220 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 220 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 240 node1/crm: Start migrating HA groups...
+noti 240 node1/crm: HA groups to rules config migration successful
+noti 240 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 240 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 240 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 240 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 240 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 240 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 240 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 260 node1/crm: Start migrating HA groups...
+noti 260 node1/crm: HA groups to rules config migration successful
+noti 260 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 260 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 260 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 260 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 260 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 260 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 260 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 280 node1/crm: Start migrating HA groups...
+noti 280 node1/crm: HA groups to rules config migration successful
+noti 280 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 280 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 280 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 280 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 280 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 280 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 280 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 300 node1/crm: Start migrating HA groups...
+noti 300 node1/crm: HA groups to rules config migration successful
+noti 300 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 300 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 300 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 300 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 300 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 300 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 300 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 320 node1/crm: Start migrating HA groups...
+noti 320 node1/crm: HA groups to rules config migration successful
+noti 320 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 320 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 320 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 320 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 320 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 320 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 320 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 340 node1/crm: Start migrating HA groups...
+noti 340 node1/crm: HA groups to rules config migration successful
+noti 340 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 340 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 340 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 340 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 340 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 340 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 340 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 360 node1/crm: Start migrating HA groups...
+noti 360 node1/crm: HA groups to rules config migration successful
+noti 360 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 360 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 360 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 360 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 360 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 360 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 360 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 380 node1/crm: Start migrating HA groups...
+noti 380 node1/crm: HA groups to rules config migration successful
+noti 380 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 380 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 380 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 380 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 380 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 380 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 380 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 400 node1/crm: Start migrating HA groups...
+noti 400 node1/crm: HA groups to rules config migration successful
+noti 400 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 400 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 400 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 400 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 400 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 400 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 400 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 420 node1/crm: Start migrating HA groups...
+noti 420 node1/crm: HA groups to rules config migration successful
+noti 420 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 420 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 420 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 420 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 420 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 420 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 420 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 440 node1/crm: Start migrating HA groups...
+noti 440 node1/crm: HA groups to rules config migration successful
+noti 440 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 440 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 440 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 440 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 440 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 440 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 440 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 460 node1/crm: Start migrating HA groups...
+noti 460 node1/crm: HA groups to rules config migration successful
+noti 460 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 460 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 460 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 460 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 460 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 460 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 460 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 480 node1/crm: Start migrating HA groups...
+noti 480 node1/crm: HA groups to rules config migration successful
+noti 480 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 480 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 480 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 480 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 480 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 480 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 480 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 500 node1/crm: Start migrating HA groups...
+noti 500 node1/crm: HA groups to rules config migration successful
+noti 500 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 500 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 500 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 500 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 500 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 500 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 500 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 520 node1/crm: Start migrating HA groups...
+noti 520 node1/crm: HA groups to rules config migration successful
+noti 520 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 520 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 520 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 520 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 520 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 520 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 520 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 540 node1/crm: Start migrating HA groups...
+noti 540 node1/crm: HA groups to rules config migration successful
+noti 540 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 540 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 540 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 540 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 540 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 540 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 540 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 560 node1/crm: Start migrating HA groups...
+noti 560 node1/crm: HA groups to rules config migration successful
+noti 560 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 560 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 560 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 560 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 560 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 560 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 560 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 580 node1/crm: Start migrating HA groups...
+noti 580 node1/crm: HA groups to rules config migration successful
+noti 580 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 580 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 580 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 580 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 580 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 580 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 580 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+noti 600 node1/crm: Start migrating HA groups...
+noti 600 node1/crm: HA groups to rules config migration successful
+noti 600 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 600 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 600 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 600 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 600 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 600 node1/crm: Node 'node3' has pve-manager version '8.4.1'
+err 600 node1/crm: Abort HA group migration: node 'node3' needs at least '9.0.0~2' to migrate HA groups
+info 620 hardware: exit simulation - done
diff --git a/src/test/test-group-migrate1/manager_status b/src/test/test-group-migrate1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-group-migrate1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-group-migrate1/service_config b/src/test/test-group-migrate1/service_config
new file mode 100644
index 0000000..a27551e
--- /dev/null
+++ b/src/test/test-group-migrate1/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "started", "group": "group1" },
+ "vm:102": { "node": "node2", "state": "started", "group": "group2" },
+ "vm:103": { "node": "node3", "state": "started", "group": "group2" }
+}
diff --git a/src/test/test-group-migrate2/README b/src/test/test-group-migrate2/README
new file mode 100644
index 0000000..8775b6c
--- /dev/null
+++ b/src/test/test-group-migrate2/README
@@ -0,0 +1,10 @@
+Test whether a service in a unrestricted group will automatically migrate back
+to a node member in case of a manual migration to a non-member node.
+
+The test scenario is:
+- vm:101 should be kept on node3
+- vm:101 is currently running on node3
+
+The expected outcome is:
+- As vm:101 is manually migrated to node2, it is migrated back to node3, as
+ node3 is a group member and has higher priority than the other nodes
diff --git a/src/test/test-group-migrate2/cmdlist b/src/test/test-group-migrate2/cmdlist
new file mode 100644
index 0000000..3bfad44
--- /dev/null
+++ b/src/test/test-group-migrate2/cmdlist
@@ -0,0 +1,3 @@
+[
+ [ "power node1 on", "power node2 on", "power node3 on"]
+]
diff --git a/src/test/test-group-migrate2/groups b/src/test/test-group-migrate2/groups
new file mode 100644
index 0000000..bad746c
--- /dev/null
+++ b/src/test/test-group-migrate2/groups
@@ -0,0 +1,7 @@
+group: group1
+ nodes node1
+ restricted 1
+
+group: group2
+ nodes node2:2,node3
+ nofailback 1
diff --git a/src/test/test-group-migrate2/hardware_status b/src/test/test-group-migrate2/hardware_status
new file mode 100644
index 0000000..5253589
--- /dev/null
+++ b/src/test/test-group-migrate2/hardware_status
@@ -0,0 +1,5 @@
+{
+ "node1": { "power": "off", "network": "off", "version": "9.0.0~2" },
+ "node2": { "power": "off", "network": "off", "version": "9.0.0~2" },
+ "node3": { "power": "off", "network": "off", "version": "9.0.0~2" }
+}
diff --git a/src/test/test-group-migrate2/log.expect b/src/test/test-group-migrate2/log.expect
new file mode 100644
index 0000000..8349f57
--- /dev/null
+++ b/src/test/test-group-migrate2/log.expect
@@ -0,0 +1,47 @@
+info 0 hardware: starting simulation
+info 20 cmdlist: execute power node1 on
+info 20 node1/crm: status change startup => wait_for_quorum
+info 20 node1/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node2 on
+info 20 node2/crm: status change startup => wait_for_quorum
+info 20 node2/lrm: status change startup => wait_for_agent_lock
+info 20 cmdlist: execute power node3 on
+info 20 node3/crm: status change startup => wait_for_quorum
+info 20 node3/lrm: status change startup => wait_for_agent_lock
+info 20 node1/crm: got lock 'ha_manager_lock'
+info 20 node1/crm: status change wait_for_quorum => master
+info 20 node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info 20 node1/crm: node 'node3': state changed from 'unknown' => 'online'
+noti 20 node1/crm: Start migrating HA groups...
+noti 20 node1/crm: HA groups to rules config migration successful
+noti 20 node1/crm: node 'node1' is in state 'online' during HA group migration.
+noti 20 node1/crm: Node 'node1' has pve-manager version '9.0.0~2'
+noti 20 node1/crm: node 'node2' is in state 'online' during HA group migration.
+noti 20 node1/crm: Node 'node2' has pve-manager version '9.0.0~2'
+noti 20 node1/crm: node 'node3' is in state 'online' during HA group migration.
+noti 20 node1/crm: Node 'node3' has pve-manager version '9.0.0~2'
+noti 20 node1/crm: HA groups to services config migration successful
+noti 20 node1/crm: HA groups config deletion successful
+noti 20 node1/crm: HA groups migration successful
+info 20 node1/crm: adding new service 'vm:101' on node 'node1'
+info 20 node1/crm: adding new service 'vm:102' on node 'node2'
+info 20 node1/crm: adding new service 'vm:103' on node 'node3'
+info 20 node1/crm: service 'vm:101': state changed from 'request_start' to 'started' (node = node1)
+info 20 node1/crm: service 'vm:102': state changed from 'request_start' to 'started' (node = node2)
+info 20 node1/crm: service 'vm:103': state changed from 'request_start' to 'started' (node = node3)
+info 21 node1/lrm: got lock 'ha_agent_node1_lock'
+info 21 node1/lrm: status change wait_for_agent_lock => active
+info 21 node1/lrm: starting service vm:101
+info 21 node1/lrm: service status vm:101 started
+info 22 node2/crm: status change wait_for_quorum => slave
+info 23 node2/lrm: got lock 'ha_agent_node2_lock'
+info 23 node2/lrm: status change wait_for_agent_lock => active
+info 23 node2/lrm: starting service vm:102
+info 23 node2/lrm: service status vm:102 started
+info 24 node3/crm: status change wait_for_quorum => slave
+info 25 node3/lrm: got lock 'ha_agent_node3_lock'
+info 25 node3/lrm: status change wait_for_agent_lock => active
+info 25 node3/lrm: starting service vm:103
+info 25 node3/lrm: service status vm:103 started
+info 620 hardware: exit simulation - done
diff --git a/src/test/test-group-migrate2/manager_status b/src/test/test-group-migrate2/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-group-migrate2/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-group-migrate2/service_config b/src/test/test-group-migrate2/service_config
new file mode 100644
index 0000000..a27551e
--- /dev/null
+++ b/src/test/test-group-migrate2/service_config
@@ -0,0 +1,5 @@
+{
+ "vm:101": { "node": "node1", "state": "started", "group": "group1" },
+ "vm:102": { "node": "node2", "state": "started", "group": "group2" },
+ "vm:103": { "node": "node3", "state": "started", "group": "group2" }
+}
--
2.39.5
More information about the pve-devel
mailing list