[pve-devel] [PATCH ha-manager v3 05/13] usage: add information about a service's assigned nodes

[Daniel Kral]

This will be used to retrieve the nodes, which a service is currently
putting load on and using their resources, when dealing with HA resource
affinity rules in select_service_node(...).

For example, a migrating service A in a negative resource affinity with
services B and C will need to block those services B and C to migrate on
both the source and target node.

This is implemented here, because the service's usage of the nodes is
currently best encoded in recompute_online_node_usage(...) and other
call sites of add_service_usage_to_node(...).

Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
 src/PVE/HA/Manager.pm      | 16 ++++++++++++----
 src/PVE/HA/Usage.pm        | 18 ++++++++++++++++++
 src/PVE/HA/Usage/Basic.pm  | 19 +++++++++++++++++++
 src/PVE/HA/Usage/Static.pm | 19 +++++++++++++++++++
 4 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 52097cf..b536c0f 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -268,6 +268,7 @@ sub recompute_online_node_usage {
                 || $state eq 'recovery'
             ) {
                 $online_node_usage->add_service_usage_to_node($sd->{node}, $sid, $sd->{node});
+                $online_node_usage->set_service_node($sid, $sd->{node});
             } elsif (
                 $state eq 'migrate'
                 || $state eq 'relocate'
@@ -275,10 +276,14 @@ sub recompute_online_node_usage {
             ) {
                 my $source = $sd->{node};
                 # count it for both, source and target as load is put on both
-                $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target)
-                    if $state ne 'request_start_balance';
-                $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target)
-                    if $online_node_usage->contains_node($target);
+                if ($state ne 'request_start_balance') {
+                    $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target);
+                    $online_node_usage->add_service_node($sid, $source);
+                }
+                if ($online_node_usage->contains_node($target)) {
+                    $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target);
+                    $online_node_usage->add_service_node($sid, $target);
+                }
             } elsif ($state eq 'stopped' || $state eq 'request_start') {
                 # do nothing
             } else {
@@ -290,6 +295,7 @@ sub recompute_online_node_usage {
                 # case a node dies, as we cannot really know if the to-be-aborted incoming migration
                 # has already cleaned up all used resources
                 $online_node_usage->add_service_usage_to_node($target, $sid, $sd->{node}, $target);
+                $online_node_usage->set_service_node($sid, $target);
             }
         }
     }
@@ -1065,6 +1071,7 @@ sub next_state_started {
 
             if ($node && ($sd->{node} ne $node)) {
                 $self->{online_node_usage}->add_service_usage_to_node($node, $sid, $sd->{node});
+                $self->{online_node_usage}->add_service_node($sid, $node);
 
                 if (defined(my $fallback = $sd->{maintenance_node})) {
                     if ($node eq $fallback) {
@@ -1193,6 +1200,7 @@ sub next_state_recovery {
 
         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
         $self->{online_node_usage}->add_service_usage_to_node($recovery_node, $sid, $recovery_node);
+        $self->{online_node_usage}->add_service_node($sid, $recovery_node);
 
         # NOTE: $sd *is normally read-only*, fencing is the exception
         $cd->{node} = $sd->{node} = $recovery_node;
diff --git a/src/PVE/HA/Usage.pm b/src/PVE/HA/Usage.pm
index 66d9572..7f4d9ca 100644
--- a/src/PVE/HA/Usage.pm
+++ b/src/PVE/HA/Usage.pm
@@ -27,6 +27,24 @@ sub list_nodes {
     die "implement in subclass";
 }
 
+sub get_service_nodes {
+    my ($self, $sid) = @_;
+
+    die "implement in subclass";
+}
+
+sub set_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    die "implement in subclass";
+}
+
+sub add_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    die "implement in subclass";
+}
+
 sub contains_node {
     my ($self, $nodename) = @_;
 
diff --git a/src/PVE/HA/Usage/Basic.pm b/src/PVE/HA/Usage/Basic.pm
index ead08c5..afe3733 100644
--- a/src/PVE/HA/Usage/Basic.pm
+++ b/src/PVE/HA/Usage/Basic.pm
@@ -11,6 +11,7 @@ sub new {
     return bless {
         nodes => {},
         haenv => $haenv,
+        'service-nodes' => {},
     }, $class;
 }
 
@@ -38,6 +39,24 @@ sub contains_node {
     return defined($self->{nodes}->{$nodename});
 }
 
+sub get_service_nodes {
+    my ($self, $sid) = @_;
+
+    return $self->{'service-nodes'}->{$sid};
+}
+
+sub set_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    $self->{'service-nodes'}->{$sid} = [$nodename];
+}
+
+sub add_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    push @{ $self->{'service-nodes'}->{$sid} }, $nodename;
+}
+
 sub add_service_usage_to_node {
     my ($self, $nodename, $sid, $service_node, $migration_target) = @_;
 
diff --git a/src/PVE/HA/Usage/Static.pm b/src/PVE/HA/Usage/Static.pm
index 061e74a..6707a54 100644
--- a/src/PVE/HA/Usage/Static.pm
+++ b/src/PVE/HA/Usage/Static.pm
@@ -22,6 +22,7 @@ sub new {
         'service-stats' => {},
         haenv => $haenv,
         scheduler => $scheduler,
+        'service-nodes' => {},
         'service-counts' => {}, # Service count on each node. Fallback if scoring calculation fails.
     }, $class;
 }
@@ -86,6 +87,24 @@ my sub get_service_usage {
     return $service_stats;
 }
 
+sub get_service_nodes {
+    my ($self, $sid) = @_;
+
+    return $self->{'service-nodes'}->{$sid};
+}
+
+sub set_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    $self->{'service-nodes'}->{$sid} = [$nodename];
+}
+
+sub add_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    push @{ $self->{'service-nodes'}->{$sid} }, $nodename;
+}
+
 sub add_service_usage_to_node {
     my ($self, $nodename, $sid, $service_node, $migration_target) = @_;
 
-- 
2.39.5