[pve-devel] [PATCH ha-manager v2 12/26] manager: apply location rules when selecting service nodes
Daniel Kral
d.kral at proxmox.com
Fri Jun 20 16:31:24 CEST 2025
Replace the HA group mechanism by replacing it with the functionally
equivalent location rules' get_location_preference(...), which enforces
the location rules defined in the rules config.
This allows the $groups parameter to be replaced with the $rules
parameter in select_service_node(...) as all behavior of the HA groups
is now encoded in $service_conf and $rules, and $rules will also be
shared with colocation rules in the next patch.
Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
changes since v1:
- NEW!
src/PVE/HA/Manager.pm | 81 +++++++-----------------------------
src/PVE/HA/Rules/Location.pm | 79 +++++++++++++++++++++++++++++++++++
src/test/test_failover1.pl | 18 +++++---
3 files changed, 107 insertions(+), 71 deletions(-)
diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 5ae8da1..00efc7c 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -10,7 +10,7 @@ use PVE::HA::Groups;
use PVE::HA::Tools ':exit_codes';
use PVE::HA::NodeStatus;
use PVE::HA::Rules;
-use PVE::HA::Rules::Location;
+use PVE::HA::Rules::Location qw(get_location_preference);
use PVE::HA::Rules::Colocation;
use PVE::HA::Usage::Basic;
use PVE::HA::Usage::Static;
@@ -115,57 +115,13 @@ sub flush_master_status {
$haenv->write_manager_status($ms);
}
-sub get_service_group {
- my ($groups, $online_node_usage, $service_conf) = @_;
-
- my $group = {};
- # add all online nodes to default group to allow try_next when no group set
- $group->{nodes}->{$_} = 1 for $online_node_usage->list_nodes();
-
- # overwrite default if service is bound to a specific group
- if (my $group_id = $service_conf->{group}) {
- $group = $groups->{ids}->{$group_id} if $groups->{ids}->{$group_id};
- }
-
- return $group;
-}
-
-# groups available nodes with their priority as group index
-sub get_node_priority_groups {
- my ($group, $online_node_usage) = @_;
-
- my $pri_groups = {};
- my $group_members = {};
- foreach my $entry (keys %{ $group->{nodes} }) {
- my ($node, $pri) = ($entry, 0);
- if ($entry =~ m/^(\S+):(\d+)$/) {
- ($node, $pri) = ($1, $2);
- }
- next if !$online_node_usage->contains_node($node); # offline
- $pri_groups->{$pri}->{$node} = 1;
- $group_members->{$node} = $pri;
- }
-
- # add non-group members to unrestricted groups (priority -1)
- if (!$group->{restricted}) {
- my $pri = -1;
- for my $node ($online_node_usage->list_nodes()) {
- next if defined($group_members->{$node});
- $pri_groups->{$pri}->{$node} = 1;
- $group_members->{$node} = -1;
- }
- }
-
- return ($pri_groups, $group_members);
-}
-
=head3 select_service_node(...)
-=head3 select_service_node($groups, $online_node_usage, $sid, $service_conf, $sd, $mode)
+=head3 select_service_node($rules, $online_node_usage, $sid, $service_conf, $sd, $mode)
Used to select the best fitting node for the service C<$sid>, with the
-configuration C<$service_conf> and state C<$sd>, according to the groups defined
-in C<$groups>, available node utilization in C<$online_node_usage>, and the
+configuration C<$service_conf> and state C<$sd>, according to the rules defined
+in C<$rules>, available node utilization in C<$online_node_usage>, and the
given C<$mode>.
The C<$mode> can be set to:
@@ -190,43 +146,36 @@ while trying to stay on the current node.
=cut
sub select_service_node {
- my ($groups, $online_node_usage, $sid, $service_conf, $sd, $mode) = @_;
+ my ($rules, $online_node_usage, $sid, $service_conf, $sd, $mode) = @_;
my ($current_node, $tried_nodes, $maintenance_fallback) =
$sd->@{qw(node failed_nodes maintenance_node)};
- my $group = get_service_group($groups, $online_node_usage, $service_conf);
+ my ($allowed_nodes, $pri_nodes) = get_location_preference($rules, $sid, $online_node_usage);
- my ($pri_groups, $group_members) = get_node_priority_groups($group, $online_node_usage);
-
- my @pri_list = sort { $b <=> $a } keys %$pri_groups;
- return undef if !scalar(@pri_list);
+ return undef if !%$pri_nodes;
# stay on current node if possible (avoids random migrations)
- if ($mode eq 'none' && $group->{nofailback} && defined($group_members->{$current_node})) {
+ if ($mode eq 'none' && !$service_conf->{failback} && $allowed_nodes->{$current_node}) {
return $current_node;
}
- # select node from top priority node list
-
- my $top_pri = $pri_list[0];
-
# try to avoid nodes where the service failed already if we want to relocate
if ($mode eq 'try-next') {
foreach my $node (@$tried_nodes) {
- delete $pri_groups->{$top_pri}->{$node};
+ delete $pri_nodes->{$node};
}
}
return $maintenance_fallback
- if defined($maintenance_fallback) && $pri_groups->{$top_pri}->{$maintenance_fallback};
+ if defined($maintenance_fallback) && $pri_nodes->{$maintenance_fallback};
- return $current_node if $mode eq 'none' && $pri_groups->{$top_pri}->{$current_node};
+ return $current_node if $mode eq 'none' && $pri_nodes->{$current_node};
my $scores = $online_node_usage->score_nodes_to_start_service($sid, $current_node);
my @nodes = sort {
$scores->{$a} <=> $scores->{$b} || $a cmp $b
- } keys %{ $pri_groups->{$top_pri} };
+ } keys %$pri_nodes;
my $found;
for (my $i = scalar(@nodes) - 1; $i >= 0; $i--) {
@@ -850,7 +799,7 @@ sub next_state_request_start {
if ($self->{crs}->{rebalance_on_request_start}) {
my $selected_node = select_service_node(
- $self->{groups},
+ $self->{rules},
$self->{online_node_usage},
$sid,
$cd,
@@ -1017,7 +966,7 @@ sub next_state_started {
}
my $node = select_service_node(
- $self->{groups},
+ $self->{rules},
$self->{online_node_usage},
$sid,
$cd,
@@ -1135,7 +1084,7 @@ sub next_state_recovery {
$self->recompute_online_node_usage(); # we want the most current node state
my $recovery_node = select_service_node(
- $self->{groups},
+ $self->{rules},
$self->{online_node_usage},
$sid,
$cd,
diff --git a/src/PVE/HA/Rules/Location.pm b/src/PVE/HA/Rules/Location.pm
index b9f76c7..4e27174 100644
--- a/src/PVE/HA/Rules/Location.pm
+++ b/src/PVE/HA/Rules/Location.pm
@@ -12,8 +12,13 @@ use PVE::Tools;
use PVE::HA::Rules;
use PVE::HA::Tools;
+use base qw(Exporter);
use base qw(PVE::HA::Rules);
+our @EXPORT_OK = qw(
+ get_location_preference
+);
+
=head1 NAME
PVE::HA::Rules::Location
@@ -207,6 +212,80 @@ __PACKAGE__->register_check(
=cut
+my $get_service_location_rule = sub {
+ my ($rules, $sid) = @_;
+
+ # with the current restriction a service can only be in one location rule
+ my $location_rule;
+ PVE::HA::Rules::foreach_rule(
+ $rules,
+ sub {
+ my ($rule) = @_;
+
+ $location_rule = dclone($rule) if !$location_rule;
+ },
+ {
+ sid => $sid,
+ type => 'location',
+ state => 'enabled',
+ },
+ );
+
+ return $location_rule;
+};
+
+=head3 get_location_preference($rules, $sid, $online_node_usage)
+
+Returns a list of two hashes, where each is a hash set of the location
+preference of C<$sid>, according to the location rules in C<$rules> and the
+available nodes in C<$online_node_usage>.
+
+The first hash is a hash set of available nodes, i.e. nodes where the
+service C<$sid> is allowed to be assigned to, and the second hash is a hash set
+of preferred nodes, i.e. nodes where the service C<$sid> should be assigned to.
+
+If there are no available nodes at all, returns C<undef>.
+
+=cut
+
+sub get_location_preference : prototype($$$) {
+ my ($rules, $sid, $online_node_usage) = @_;
+
+ my $location_rule = $get_service_location_rule->($rules, $sid);
+
+ # default to a location rule with all available nodes
+ if (!$location_rule) {
+ for my $node ($online_node_usage->list_nodes()) {
+ $location_rule->{nodes}->{$node} = { priority => 0 };
+ }
+ }
+
+ # add remaining nodes with low priority for non-strict location rules
+ if (!$location_rule->{strict}) {
+ for my $node ($online_node_usage->list_nodes()) {
+ next if defined($location_rule->{nodes}->{$node});
+
+ $location_rule->{nodes}->{$node} = { priority => -1 };
+ }
+ }
+
+ my $allowed_nodes = {};
+ my $prioritized_nodes = {};
+
+ while (my ($node, $props) = each %{ $location_rule->{nodes} }) {
+ next if !$online_node_usage->contains_node($node); # node is offline
+
+ $allowed_nodes->{$node} = 1;
+ $prioritized_nodes->{ $props->{priority} }->{$node} = 1;
+ }
+
+ my $preferred_nodes = {};
+ my $highest_priority = (sort { $b <=> $a } keys %$prioritized_nodes)[0];
+ $preferred_nodes = $prioritized_nodes->{$highest_priority} if defined($highest_priority);
+
+ return ($allowed_nodes, $preferred_nodes);
+}
+
# Remove location rules from rules config
# TODO PVE 10: Can be removed if use-location-rules feature flag is not needed anymore
sub delete_location_rules {
diff --git a/src/test/test_failover1.pl b/src/test/test_failover1.pl
index 90bd61a..412d88d 100755
--- a/src/test/test_failover1.pl
+++ b/src/test/test_failover1.pl
@@ -4,12 +4,21 @@ use strict;
use warnings;
use lib '..';
-use PVE::HA::Groups;
use PVE::HA::Manager;
use PVE::HA::Usage::Basic;
-my $groups = PVE::HA::Groups->parse_config("groups.tmp", <<EOD);
-group: prefer_node1
+use PVE::HA::Rules;
+use PVE::HA::Rules::Location;
+use PVE::HA::Rules::Colocation;
+
+PVE::HA::Rules::Location->register();
+PVE::HA::Rules::Colocation->register();
+
+PVE::HA::Rules->init(property_isolation => 1);
+
+my $rules = PVE::HA::Rules->parse_config("rules.tmp", <<EOD);
+location: prefer_node1
+ services vm:111
nodes node1
EOD
@@ -21,7 +30,6 @@ $online_node_usage->add_node("node3");
my $service_conf = {
node => 'node1',
- group => 'prefer_node1',
failback => 1,
};
@@ -37,7 +45,7 @@ sub test {
my $select_mode = $try_next ? 'try-next' : 'none';
my $node = PVE::HA::Manager::select_service_node(
- $groups, $online_node_usage, "vm:111", $service_conf, $sd, $select_mode,
+ $rules, $online_node_usage, "vm:111", $service_conf, $sd, $select_mode,
);
my (undef, undef, $line) = caller();
--
2.39.5
More information about the pve-devel
mailing list