[pve-devel] [PATCH ha-manager v2 14/26] manager: apply colocation rules when selecting service nodes

[Daniel Kral]

Add a mechanism to the node selection subroutine, which enforces the
colocation rules defined in the rules config.

The algorithm makes in-place changes to the set of nodes in such a way,
that the final set contains only the nodes where the colocation rules
allow the service to run on, depending on the affinity type of the
colocation rules.

The service's failback property also slightly changes meaning because
now it also controls how services behave for colocation rules, not only
location rules.

Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
changes since v1:
    - added documentation
    - moved apply helpers from Manager.pm into Colocation rule plugin
    - dropped loose colocations, so I could shrink down the apply
      helpers to only a few lines
    - dropped `get_colocated_services(...)` from this patch (which will
      be introduced in another version in a later patch), and merged its
      logic into `get_colocation_preference(...)`
    - fix bug when positively colocated services are on different nodes
      (e.g. when newly creating a rule for these), then they could still
      favor their current node, because then multiple nodes are in the
      $together hash set. For now, just select for all of them the node
      which is the most populated by all other pos. colocated services;
      this can be improved in a follow-up to check which node has the
      resources for all of them, for example
    - introduce `is_allowed_on_node(...)` helper to check in 'none' mode
      whether the current node is compliant with the colocation rules

 src/PVE/HA/Manager.pm          |  15 +++-
 src/PVE/HA/Resources.pm        |   3 +-
 src/PVE/HA/Rules/Colocation.pm | 151 +++++++++++++++++++++++++++++++++
 3 files changed, 166 insertions(+), 3 deletions(-)

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 4c7228e..a69898b 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -11,7 +11,8 @@ use PVE::HA::Tools ':exit_codes';
 use PVE::HA::NodeStatus;
 use PVE::HA::Rules;
 use PVE::HA::Rules::Location qw(get_location_preference);
-use PVE::HA::Rules::Colocation;
+use PVE::HA::Rules::Colocation
+    qw(get_colocation_preference apply_positive_colocation_rules apply_negative_colocation_rules);
 use PVE::HA::Usage::Basic;
 use PVE::HA::Usage::Static;
 
@@ -155,8 +156,15 @@ sub select_service_node {
 
     return undef if !%$pri_nodes;
 
+    my ($together, $separate) = get_colocation_preference($rules, $sid, $online_node_usage);
+
     # stay on current node if possible (avoids random migrations)
-    if ($mode eq 'none' && !$service_conf->{failback} && $allowed_nodes->{$current_node}) {
+    if (
+        $mode eq 'none'
+        && !$service_conf->{failback}
+        && $allowed_nodes->{$current_node}
+        && PVE::HA::Rules::Colocation::is_allowed_on_node($together, $separate, $current_node)
+    ) {
         return $current_node;
     }
 
@@ -167,6 +175,9 @@ sub select_service_node {
         }
     }
 
+    apply_positive_colocation_rules($together, $pri_nodes);
+    apply_negative_colocation_rules($separate, $pri_nodes);
+
     return $maintenance_fallback
         if defined($maintenance_fallback) && $pri_nodes->{$maintenance_fallback};
 
diff --git a/src/PVE/HA/Resources.pm b/src/PVE/HA/Resources.pm
index 90410a9..f8aad35 100644
--- a/src/PVE/HA/Resources.pm
+++ b/src/PVE/HA/Resources.pm
@@ -65,7 +65,8 @@ EODESC
         failback => {
             description => "Automatically migrate service to the node with the highest priority"
                 . " according to their location rules, if a node with a higher priority than the"
-                . " current node comes online.",
+                . " current node comes online, or migrate to the node, which doesn't violate any"
+                . " colocation rule.",
             type => 'boolean',
             optional => 1,
             default => 1,
diff --git a/src/PVE/HA/Rules/Colocation.pm b/src/PVE/HA/Rules/Colocation.pm
index 0539eb3..190478e 100644
--- a/src/PVE/HA/Rules/Colocation.pm
+++ b/src/PVE/HA/Rules/Colocation.pm
@@ -7,8 +7,15 @@ use PVE::HashTools;
 
 use PVE::HA::Rules;
 
+use base qw(Exporter);
 use base qw(PVE::HA::Rules);
 
+our @EXPORT_OK = qw(
+    get_colocation_preference
+    apply_positive_colocation_rules
+    apply_negative_colocation_rules
+);
+
 =head1 NAME
 
 PVE::HA::Rules::Colocation - Colocation Plugin for HA Rules
@@ -284,4 +291,148 @@ sub plugin_canonicalize {
     merge_connected_positive_colocation_rules($rules, $args->{positive_rules});
 }
 
+=head1 COLOCATION RULE HELPERS
+
+=cut
+
+=head3 get_colocation_preference($rules, $sid, $online_node_usage)
+
+Returns a list of two hashes, where the first describes the positive colocation
+preference and the second describes the negative colocation preference for
+C<$sid> according to the colocation rules in C<$rules> and the service
+locations in C<$online_node_usage>.
+
+For the positive colocation preference of a service C<$sid>, each element in the
+hash represents an online node, where other positively colocated services are
+already running, and how many of them. That is, each element represents a node,
+where the service must be.
+
+For the negative colocation preference of a service C<$sid>, each element in the
+hash represents an online node, where other negatively colocated services are
+already running. That is, each element represents a node, where the service must
+not be.
+
+For example, if there are already three services running, which the service
+C<$sid> is in a positive colocation with, and two running services, which the
+service C<$sid> is in a negative colocation relationship with, the returned
+value will be:
+
+    {
+        together => {
+            node2 => 3
+        },
+        separate => {
+            node1 => 1,
+            node3 => 1
+        }
+    }
+
+=cut
+
+sub get_colocation_preference : prototype($$$) {
+    my ($rules, $sid, $online_node_usage) = @_;
+
+    my $together = {};
+    my $separate = {};
+
+    PVE::HA::Rules::foreach_rule(
+        $rules,
+        sub {
+            my ($rule) = @_;
+
+            for my $csid (keys %{ $rule->{services} }) {
+                next if $csid eq $sid;
+
+                my $nodes = $online_node_usage->get_service_nodes($csid);
+
+                next if !$nodes || !@$nodes; # skip unassigned nodes
+
+                if ($rule->{affinity} eq 'together') {
+                    $together->{$_}++ for @$nodes;
+                } elsif ($rule->{affinity} eq 'separate') {
+                    $separate->{$_} = 1 for @$nodes;
+                } else {
+                    die "unimplemented colocation affinity type $rule->{affinity}\n";
+                }
+            }
+        },
+        {
+            sid => $sid,
+            type => 'colocation',
+            state => 'enabled',
+        },
+    );
+
+    return ($together, $separate);
+}
+
+=head3 is_allowed_on_node($together, $separate, $node)
+
+Checks whether the colocation preference hashes C<$together> or C<$separate>
+describe colocation relation, where for C<$together> the C<$node> must be
+selected, or for C<$separate> the node C<$node> must be avoided.
+
+=cut
+
+sub is_allowed_on_node : prototype($$$) {
+    my ($together, $separate, $node) = @_;
+
+    return $together->{$node} || !$separate->{$node};
+}
+
+=head3 apply_positive_colocation_rules($together, $allowed_nodes)
+
+Applies the positive colocation preference C<$together> on the allowed node
+hash set C<$allowed_nodes> by modifying it directly.
+
+Positive colocation means keeping services together on a single node and
+therefore minimizing the separation of services.
+
+The allowed node hash set C<$allowed_nodes> is expected to contain all nodes,
+which are available to the service this helper is called for, i.e. each node
+is currently online, available according to other location constraints, and the
+service has not failed running there yet.
+
+=cut
+
+sub apply_positive_colocation_rules : prototype($$) {
+    my ($together, $allowed_nodes) = @_;
+
+    my @possible_nodes = sort keys $together->%*
+        or return; # nothing to do if there are no positive colocation preferences
+
+    # select the most populated node as the target node for positive colocations
+    @possible_nodes = sort { $together->{$b} <=> $together->{$a} } @possible_nodes;
+    my $majority_node = $possible_nodes[0];
+
+    for my $node (keys %$allowed_nodes) {
+        delete $allowed_nodes->{$node} if $node ne $majority_node;
+    }
+}
+
+=head3 apply_negative_colocation_rules($separate, $allowed_nodes)
+
+Applies the negative colocation preference C<$separate> on the allowed node
+hash set C<$allowed_nodes> by modifying it directly.
+
+Negative colocation means keeping services separate on multiple nodes and
+therefore maximizing the separation of services.
+
+The allowed node hash set C<$allowed_nodes> is expected to contain all nodes,
+which are available to the service this helper is called for, i.e. each node
+is currently online, available according to other location constraints, and the
+service has not failed running there yet.
+
+=cut
+
+sub apply_negative_colocation_rules : prototype($$) {
+    my ($separate, $allowed_nodes) = @_;
+
+    my $forbidden_nodes = { $separate->%* };
+
+    for my $node (keys %$forbidden_nodes) {
+        delete $allowed_nodes->{$node};
+    }
+}
+
 1;
-- 
2.39.5