[pve-devel] [PATCH ha-manager 09/15] manager: apply colocation rules when selecting service nodes
Daniel Kral
d.kral at proxmox.com
Tue Mar 25 16:12:48 CET 2025
Add a mechanism to the node selection subroutine, which enforces the
colocation rules defined in the rules config.
The algorithm manipulates the set of nodes directly, which the service
is allowed to run on, depending on the type and strictness of the
colocation rules, if there are any.
This makes it depend on the prior removal of any nodes, which are
unavailable (i.e. offline, unreachable, or weren't able to start the
service in previous tries) or are not allowed to be run on otherwise
(i.e. HA group node restrictions) to function correctly.
Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
src/PVE/HA/Manager.pm | 203 ++++++++++++++++++++++++++++++++++++-
src/test/test_failover1.pl | 4 +-
2 files changed, 205 insertions(+), 2 deletions(-)
diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 8f2ab3d..79b6555 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -157,8 +157,201 @@ sub get_node_priority_groups {
return ($pri_groups, $group_members);
}
+=head3 get_colocated_services($rules, $sid, $online_node_usage)
+
+Returns a hash map of all services, which are specified as being in a positive
+or negative colocation in C<$rules> with the given service with id C<$sid>.
+
+Each service entry consists of the type of colocation, strictness of colocation
+and the node the service is currently assigned to, if any, according to
+C<$online_node_usage>.
+
+For example, a service C<'vm:101'> being strictly colocated together (positive)
+with two other services C<'vm:102'> and C<'vm:103'> and loosely colocated
+separate with another service C<'vm:104'> results in the hash map:
+
+ {
+ 'vm:102' => {
+ affinity => 'together',
+ strict => 1,
+ node => 'node2'
+ },
+ 'vm:103' => {
+ affinity => 'together',
+ strict => 1,
+ node => 'node2'
+ },
+ 'vm:104' => {
+ affinity => 'separate',
+ strict => 0,
+ node => undef
+ }
+ }
+
+=cut
+
+sub get_colocated_services {
+ my ($rules, $sid, $online_node_usage) = @_;
+
+ my $services = {};
+
+ PVE::HA::Rules::Colocation::foreach_colocation_rule($rules, sub {
+ my ($rule) = @_;
+
+ for my $csid (sort keys %{$rule->{services}}) {
+ next if $csid eq $sid;
+
+ $services->{$csid} = {
+ node => $online_node_usage->get_service_node($csid),
+ affinity => $rule->{affinity},
+ strict => $rule->{strict},
+ };
+ }
+ }, {
+ sid => $sid,
+ });
+
+ return $services;
+}
+
+=head3 get_colocation_preference($rules, $sid, $online_node_usage)
+
+Returns a list of two hashes, where each is a hash map of the colocation
+preference of C<$sid>, according to the colocation rules in C<$rules> and the
+service locations in C<$online_node_usage>.
+
+The first hash is the positive colocation preference, where each element
+represents properties for how much C<$sid> prefers to be on the node.
+Currently, this is a binary C<$strict> field, which means either it should be
+there (C<0>) or must be there (C<1>).
+
+The second hash is the negative colocation preference, where each element
+represents properties for how much C<$sid> prefers not to be on the node.
+Currently, this is a binary C<$strict> field, which means either it should not
+be there (C<0>) or must not be there (C<1>).
+
+=cut
+
+sub get_colocation_preference {
+ my ($rules, $sid, $online_node_usage) = @_;
+
+ my $services = get_colocated_services($rules, $sid, $online_node_usage);
+
+ my $together = {};
+ my $separate = {};
+
+ for my $service (values %$services) {
+ my $node = $service->{node};
+
+ next if !$node;
+
+ my $node_set = $service->{affinity} eq 'together' ? $together : $separate;
+ $node_set->{$node}->{strict} = $node_set->{$node}->{strict} || $service->{strict};
+ }
+
+ return ($together, $separate);
+}
+
+=head3 apply_positive_colocation_rules($together, $allowed_nodes)
+
+Applies the positive colocation preference C<$together> on the allowed node
+hash set C<$allowed_nodes> directly.
+
+Positive colocation means keeping services together on a single node, and
+therefore minimizing the separation of services.
+
+The allowed node hash set C<$allowed_nodes> is expected to contain any node,
+which is available to the service, i.e. each node is currently online, is
+available according to other location constraints, and the service has not
+failed running there yet.
+
+=cut
+
+sub apply_positive_colocation_rules {
+ my ($together, $allowed_nodes) = @_;
+
+ return if scalar(keys %$together) < 1;
+
+ my $mandatory_nodes = {};
+ my $possible_nodes = PVE::HA::Tools::intersect($allowed_nodes, $together);
+
+ for my $node (sort keys %$together) {
+ $mandatory_nodes->{$node} = 1 if $together->{$node}->{strict};
+ }
+
+ if (scalar keys %$mandatory_nodes) {
+ # limit to only the nodes the service must be on.
+ for my $node (keys %$allowed_nodes) {
+ next if exists($mandatory_nodes->{$node});
+
+ delete $allowed_nodes->{$node};
+ }
+ } elsif (scalar keys %$possible_nodes) {
+ # limit to the possible nodes the service should be on, if there are any.
+ for my $node (keys %$allowed_nodes) {
+ next if exists($possible_nodes->{$node});
+
+ delete $allowed_nodes->{$node};
+ }
+ }
+}
+
+=head3 apply_negative_colocation_rules($separate, $allowed_nodes)
+
+Applies the negative colocation preference C<$separate> on the allowed node
+hash set C<$allowed_nodes> directly.
+
+Negative colocation means keeping services separate on multiple nodes, and
+therefore maximizing the separation of services.
+
+The allowed node hash set C<$allowed_nodes> is expected to contain any node,
+which is available to the service, i.e. each node is currently online, is
+available according to other location constraints, and the service has not
+failed running there yet.
+
+=cut
+
+sub apply_negative_colocation_rules {
+ my ($separate, $allowed_nodes) = @_;
+
+ return if scalar(keys %$separate) < 1;
+
+ my $mandatory_nodes = {};
+ my $possible_nodes = PVE::HA::Tools::set_difference($allowed_nodes, $separate);
+
+ for my $node (sort keys %$separate) {
+ $mandatory_nodes->{$node} = 1 if $separate->{$node}->{strict};
+ }
+
+ if (scalar keys %$mandatory_nodes) {
+ # limit to the nodes the service must not be on.
+ for my $node (keys %$allowed_nodes) {
+ next if !exists($mandatory_nodes->{$node});
+
+ delete $allowed_nodes->{$node};
+ }
+ } elsif (scalar keys %$possible_nodes) {
+ # limit to the nodes the service should not be on, if any.
+ for my $node (keys %$allowed_nodes) {
+ next if exists($possible_nodes->{$node});
+
+ delete $allowed_nodes->{$node};
+ }
+ }
+}
+
+sub apply_colocation_rules {
+ my ($rules, $sid, $allowed_nodes, $online_node_usage) = @_;
+
+ my ($together, $separate) = get_colocation_preference($rules, $sid, $online_node_usage);
+
+ apply_positive_colocation_rules($together, $allowed_nodes);
+ apply_negative_colocation_rules($separate, $allowed_nodes);
+}
+
sub select_service_node {
- my ($groups, $online_node_usage, $sid, $service_conf, $current_node, $try_next, $tried_nodes, $maintenance_fallback, $best_scored) = @_;
+ # TODO Cleanup this signature post-RFC
+ my ($rules, $groups, $online_node_usage, $sid, $service_conf, $current_node, $try_next, $tried_nodes, $maintenance_fallback, $best_scored) = @_;
my $group = get_service_group($groups, $online_node_usage, $service_conf);
@@ -189,6 +382,8 @@ sub select_service_node {
return $current_node if (!$try_next && !$best_scored) && $pri_nodes->{$current_node};
+ apply_colocation_rules($rules, $sid, $pri_nodes, $online_node_usage);
+
my $scores = $online_node_usage->score_nodes_to_start_service($sid, $current_node);
my @nodes = sort {
$scores->{$a} <=> $scores->{$b} || $a cmp $b
@@ -758,6 +953,7 @@ sub next_state_request_start {
if ($self->{crs}->{rebalance_on_request_start}) {
my $selected_node = select_service_node(
+ $self->{rules},
$self->{groups},
$self->{online_node_usage},
$sid,
@@ -771,6 +967,9 @@ sub next_state_request_start {
my $select_text = $selected_node ne $current_node ? 'new' : 'current';
$haenv->log('info', "service $sid: re-balance selected $select_text node $selected_node for startup");
+ # TODO It would be better if this information would be retrieved from $ss/$sd post-RFC
+ $self->{online_node_usage}->pin_service_node($sid, $selected_node);
+
if ($selected_node ne $current_node) {
$change_service_state->($self, $sid, 'request_start_balance', node => $current_node, target => $selected_node);
return;
@@ -898,6 +1097,7 @@ sub next_state_started {
}
my $node = select_service_node(
+ $self->{rules},
$self->{groups},
$self->{online_node_usage},
$sid,
@@ -1004,6 +1204,7 @@ sub next_state_recovery {
$self->recompute_online_node_usage(); # we want the most current node state
my $recovery_node = select_service_node(
+ $self->{rules},
$self->{groups},
$self->{online_node_usage},
$sid,
diff --git a/src/test/test_failover1.pl b/src/test/test_failover1.pl
index 308eab3..4c84fbd 100755
--- a/src/test/test_failover1.pl
+++ b/src/test/test_failover1.pl
@@ -8,6 +8,8 @@ use PVE::HA::Groups;
use PVE::HA::Manager;
use PVE::HA::Usage::Basic;
+my $rules = {};
+
my $groups = PVE::HA::Groups->parse_config("groups.tmp", <<EOD);
group: prefer_node1
nodes node1
@@ -31,7 +33,7 @@ sub test {
my ($expected_node, $try_next) = @_;
my $node = PVE::HA::Manager::select_service_node
- ($groups, $online_node_usage, "vm:111", $service_conf, $current_node, $try_next);
+ ($rules, $groups, $online_node_usage, "vm:111", $service_conf, $current_node, $try_next);
my (undef, undef, $line) = caller();
die "unexpected result: $node != ${expected_node} at line $line\n"
--
2.39.5
More information about the pve-devel
mailing list