[pve-devel] [PATCH ha-manager v2 13/26] usage: add information about a service's assigned nodes

[Daniel Kral]

This will be used to retrieve the nodes, which a service is currently
putting load on and using their resources, when dealing with colocation
rules in select_service_node(...). For example, a migrating service in a
negative colocation will need to block other negatively colocated
services to migrate on both the source and target node.

This is implemented here, because the service's usage of the nodes is
currently best encoded in recompute_online_node_usage(...) and other
call sites of add_service_usage_to_node(...).

Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
changes since v1:
    - let these be in Usage as it would introduce a second/third source
      of truth where the services are supposed to be; this should
      possibly be refactored in a follow-up (e.g. when making service
      state/node load changes more granular)
    - use services-nodes key for both Usage implementations
    - replace `pin_service_node(...)` with `set_service_node(...)`
    - introduce `add_service_node(...)` and allow multiple nodes in the
      services-nodes hash
    - make adds to the services-nodes hash more explicit with direct
      calls to `{add,set}_service_node(...)` instead of being in
      `add_service_usage_to_node(...)`

 src/PVE/HA/Manager.pm      | 17 +++++++++++++----
 src/PVE/HA/Usage.pm        | 18 ++++++++++++++++++
 src/PVE/HA/Usage/Basic.pm  | 19 +++++++++++++++++++
 src/PVE/HA/Usage/Static.pm | 19 +++++++++++++++++++
 4 files changed, 69 insertions(+), 4 deletions(-)

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 00efc7c..4c7228e 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -258,6 +258,7 @@ sub recompute_online_node_usage {
         my $sd = $self->{ss}->{$sid};
         my $state = $sd->{state};
         my $target = $sd->{target}; # optional
+
         if ($online_node_usage->contains_node($sd->{node})) {
             if (
                 $state eq 'started'
@@ -268,6 +269,7 @@ sub recompute_online_node_usage {
                 || $state eq 'recovery'
             ) {
                 $online_node_usage->add_service_usage_to_node($sd->{node}, $sid, $sd->{node});
+                $online_node_usage->set_service_node($sid, $sd->{node});
             } elsif (
                 $state eq 'migrate'
                 || $state eq 'relocate'
@@ -275,10 +277,14 @@ sub recompute_online_node_usage {
             ) {
                 my $source = $sd->{node};
                 # count it for both, source and target as load is put on both
-                $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target)
-                    if $state ne 'request_start_balance';
-                $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target)
-                    if $online_node_usage->contains_node($target);
+                if ($state ne 'request_start_balance') {
+                    $online_node_usage->add_service_usage_to_node($source, $sid, $source, $target);
+                    $online_node_usage->add_service_node($sid, $source);
+                }
+                if ($online_node_usage->contains_node($target)) {
+                    $online_node_usage->add_service_usage_to_node($target, $sid, $source, $target);
+                    $online_node_usage->add_service_node($sid, $target);
+                }
             } elsif ($state eq 'stopped' || $state eq 'request_start') {
                 # do nothing
             } else {
@@ -290,6 +296,7 @@ sub recompute_online_node_usage {
                 # case a node dies, as we cannot really know if the to-be-aborted incoming migration
                 # has already cleaned up all used resources
                 $online_node_usage->add_service_usage_to_node($target, $sid, $sd->{node}, $target);
+                $online_node_usage->set_service_node($sid, $target);
             }
         }
     }
@@ -976,6 +983,7 @@ sub next_state_started {
 
             if ($node && ($sd->{node} ne $node)) {
                 $self->{online_node_usage}->add_service_usage_to_node($node, $sid, $sd->{node});
+                $self->{online_node_usage}->add_service_node($sid, $node);
 
                 if (defined(my $fallback = $sd->{maintenance_node})) {
                     if ($node eq $fallback) {
@@ -1104,6 +1112,7 @@ sub next_state_recovery {
 
         $haenv->steal_service($sid, $sd->{node}, $recovery_node);
         $self->{online_node_usage}->add_service_usage_to_node($recovery_node, $sid, $recovery_node);
+        $self->{online_node_usage}->add_service_node($sid, $recovery_node);
 
         # NOTE: $sd *is normally read-only*, fencing is the exception
         $cd->{node} = $sd->{node} = $recovery_node;
diff --git a/src/PVE/HA/Usage.pm b/src/PVE/HA/Usage.pm
index 66d9572..7f4d9ca 100644
--- a/src/PVE/HA/Usage.pm
+++ b/src/PVE/HA/Usage.pm
@@ -27,6 +27,24 @@ sub list_nodes {
     die "implement in subclass";
 }
 
+sub get_service_nodes {
+    my ($self, $sid) = @_;
+
+    die "implement in subclass";
+}
+
+sub set_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    die "implement in subclass";
+}
+
+sub add_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    die "implement in subclass";
+}
+
 sub contains_node {
     my ($self, $nodename) = @_;
 
diff --git a/src/PVE/HA/Usage/Basic.pm b/src/PVE/HA/Usage/Basic.pm
index ead08c5..afe3733 100644
--- a/src/PVE/HA/Usage/Basic.pm
+++ b/src/PVE/HA/Usage/Basic.pm
@@ -11,6 +11,7 @@ sub new {
     return bless {
         nodes => {},
         haenv => $haenv,
+        'service-nodes' => {},
     }, $class;
 }
 
@@ -38,6 +39,24 @@ sub contains_node {
     return defined($self->{nodes}->{$nodename});
 }
 
+sub get_service_nodes {
+    my ($self, $sid) = @_;
+
+    return $self->{'service-nodes'}->{$sid};
+}
+
+sub set_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    $self->{'service-nodes'}->{$sid} = [$nodename];
+}
+
+sub add_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    push @{ $self->{'service-nodes'}->{$sid} }, $nodename;
+}
+
 sub add_service_usage_to_node {
     my ($self, $nodename, $sid, $service_node, $migration_target) = @_;
 
diff --git a/src/PVE/HA/Usage/Static.pm b/src/PVE/HA/Usage/Static.pm
index 061e74a..6707a54 100644
--- a/src/PVE/HA/Usage/Static.pm
+++ b/src/PVE/HA/Usage/Static.pm
@@ -22,6 +22,7 @@ sub new {
         'service-stats' => {},
         haenv => $haenv,
         scheduler => $scheduler,
+        'service-nodes' => {},
         'service-counts' => {}, # Service count on each node. Fallback if scoring calculation fails.
     }, $class;
 }
@@ -86,6 +87,24 @@ my sub get_service_usage {
     return $service_stats;
 }
 
+sub get_service_nodes {
+    my ($self, $sid) = @_;
+
+    return $self->{'service-nodes'}->{$sid};
+}
+
+sub set_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    $self->{'service-nodes'}->{$sid} = [$nodename];
+}
+
+sub add_service_node {
+    my ($self, $sid, $nodename) = @_;
+
+    push @{ $self->{'service-nodes'}->{$sid} }, $nodename;
+}
+
 sub add_service_usage_to_node {
     my ($self, $nodename, $sid, $service_node, $migration_target) = @_;
 
-- 
2.39.5