[pve-devel] [PATCH ha-manager v3 04/15] rules: introduce node affinity rule plugin
Daniel Kral
d.kral at proxmox.com
Fri Jul 4 20:16:44 CEST 2025
Introduce the node affinity rule plugin to allow users to specify node
affinity constraints for independent HA resources.
Node affinity rules must specify one or more HA resources, one or more
nodes with optional priorities (the default is 0), and a strictness,
which is either
* 0 (non-strict): HA resources SHOULD be on one of the rules' nodes, or
* 1 (strict): HA resources MUST be on one of the rules' nodes, or
The initial implementation restricts node affinity rules to only specify
a single HA resource once across all node affinity rules, else these
node affinity rules will not be applied.
This makes node affinity rules structurally equivalent to HA groups with
the exception of the "failback" option, which will be moved to the HA
resource config in an upcoming patch.
The HA resources property is added to the rules base plugin as it will
also planned to be used by other rule plugins, e.g., the resource
affinity rule plugin.
Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
debian/pve-ha-manager.install | 1 +
src/PVE/HA/Makefile | 1 +
src/PVE/HA/Rules.pm | 29 ++++-
src/PVE/HA/Rules/Makefile | 6 +
src/PVE/HA/Rules/NodeAffinity.pm | 213 +++++++++++++++++++++++++++++++
src/PVE/HA/Tools.pm | 24 ++++
6 files changed, 272 insertions(+), 2 deletions(-)
create mode 100644 src/PVE/HA/Rules/Makefile
create mode 100644 src/PVE/HA/Rules/NodeAffinity.pm
diff --git a/debian/pve-ha-manager.install b/debian/pve-ha-manager.install
index 9bbd375..7462663 100644
--- a/debian/pve-ha-manager.install
+++ b/debian/pve-ha-manager.install
@@ -33,6 +33,7 @@
/usr/share/perl5/PVE/HA/Resources/PVECT.pm
/usr/share/perl5/PVE/HA/Resources/PVEVM.pm
/usr/share/perl5/PVE/HA/Rules.pm
+/usr/share/perl5/PVE/HA/Rules/NodeAffinity.pm
/usr/share/perl5/PVE/HA/Tools.pm
/usr/share/perl5/PVE/HA/Usage.pm
/usr/share/perl5/PVE/HA/Usage/Basic.pm
diff --git a/src/PVE/HA/Makefile b/src/PVE/HA/Makefile
index 489cbc0..e386cbf 100644
--- a/src/PVE/HA/Makefile
+++ b/src/PVE/HA/Makefile
@@ -8,6 +8,7 @@ install:
install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA
for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/$$i; done
make -C Resources install
+ make -C Rules install
make -C Usage install
make -C Env install
diff --git a/src/PVE/HA/Rules.pm b/src/PVE/HA/Rules.pm
index d786669..bda0b5d 100644
--- a/src/PVE/HA/Rules.pm
+++ b/src/PVE/HA/Rules.pm
@@ -109,6 +109,13 @@ my $defaultData = {
type => 'boolean',
optional => 1,
},
+ resources => get_standard_option(
+ 'pve-ha-resource-id-list',
+ {
+ completion => \&PVE::HA::Tools::complete_sid,
+ optional => 0,
+ },
+ ),
comment => {
description => "HA rule description.",
type => 'string',
@@ -145,7 +152,17 @@ sub decode_plugin_value {
sub decode_value {
my ($class, $type, $key, $value) = @_;
- if ($key eq 'comment') {
+ if ($key eq 'resources') {
+ my $res = {};
+
+ for my $sid (PVE::Tools::split_list($value)) {
+ if (PVE::HA::Tools::pve_verify_ha_resource_id($sid)) {
+ $res->{$sid} = 1;
+ }
+ }
+
+ return $res;
+ } elsif ($key eq 'comment') {
return PVE::Tools::decode_text($value);
}
@@ -176,7 +193,11 @@ sub encode_plugin_value {
sub encode_value {
my ($class, $type, $key, $value) = @_;
- if ($key eq 'comment') {
+ if ($key eq 'resources') {
+ PVE::HA::Tools::pve_verify_ha_resource_id($_) for keys %$value;
+
+ return join(',', sort keys %$value);
+ } elsif ($key eq 'comment') {
return PVE::Tools::encode_text($value);
}
@@ -383,6 +404,8 @@ The filter properties for C<$opts> are:
=over
+=item C<$sid>: Limits C<$rules> to those which contain the given resource C<$sid>.
+
=item C<$type>: Limits C<$rules> to those which are of rule type C<$type>.
=item C<$exclude_disabled_rules>: Limits C<$rules> to those which are enabled.
@@ -394,6 +417,7 @@ The filter properties for C<$opts> are:
sub foreach_rule : prototype($$;$) {
my ($rules, $func, $opts) = @_;
+ my $sid = $opts->{sid};
my $type = $opts->{type};
my $exclude_disabled_rules = $opts->{exclude_disabled_rules};
@@ -405,6 +429,7 @@ sub foreach_rule : prototype($$;$) {
my $rule = $rules->{ids}->{$ruleid};
next if !$rule; # skip invalid rules
+ next if defined($sid) && !defined($rule->{resources}->{$sid});
next if defined($type) && $rule->{type} ne $type;
next if $exclude_disabled_rules && exists($rule->{disable});
diff --git a/src/PVE/HA/Rules/Makefile b/src/PVE/HA/Rules/Makefile
new file mode 100644
index 0000000..dfef257
--- /dev/null
+++ b/src/PVE/HA/Rules/Makefile
@@ -0,0 +1,6 @@
+SOURCES=NodeAffinity.pm
+
+.PHONY: install
+install:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA/Rules
+ for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/Rules/$$i; done
diff --git a/src/PVE/HA/Rules/NodeAffinity.pm b/src/PVE/HA/Rules/NodeAffinity.pm
new file mode 100644
index 0000000..2b3d739
--- /dev/null
+++ b/src/PVE/HA/Rules/NodeAffinity.pm
@@ -0,0 +1,213 @@
+package PVE::HA::Rules::NodeAffinity;
+
+use strict;
+use warnings;
+
+use Storable qw(dclone);
+
+use PVE::Cluster;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::Tools;
+
+use PVE::HA::Rules;
+use PVE::HA::Tools;
+
+use base qw(PVE::HA::Rules);
+
+=head1 NAME
+
+PVE::HA::Rules::NodeAffinity
+
+=head1 DESCRIPTION
+
+This package provides the capability to specify and apply rules, which put
+affinity constraints between a set of HA resources and a set of nodes.
+
+HA Node Affinity rules can be either C<'non-strict'> or C<'strict'>:
+
+=over
+
+=item C<'non-strict'>
+
+Non-strict node affinity rules SHOULD be applied if possible.
+
+That is, HA resources SHOULD prefer to be on the defined nodes, but may fall
+back to other nodes, if none of the defined nodes are available.
+
+=item C<'strict'>
+
+Strict node affinity rules MUST be applied.
+
+That is, HA resources MUST prefer to be on the defined nodes. In other words,
+these HA resources are restricted to the defined nodes and may not run on any
+other node.
+
+=back
+
+=cut
+
+sub type {
+ return 'node-affinity';
+}
+
+sub properties {
+ return {
+ nodes => get_standard_option(
+ 'pve-ha-group-node-list',
+ {
+ completion => \&PVE::Cluster::get_nodelist,
+ optional => 0,
+ },
+ ),
+ strict => {
+ description => "Describes whether the node affinity rule is strict or non-strict.",
+ verbose_description => <<EODESC,
+Describes whether the node affinity rule is strict or non-strict.
+
+A non-strict node affinity rule makes resources prefer to be on the defined nodes.
+If none of the defined nodes are available, the resource may run on any other node.
+
+A strict node affinity rule makes resources be restricted to the defined nodes. If
+none of the defined nodes are available, the resource will be stopped.
+EODESC
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ },
+ };
+}
+
+sub options {
+ return {
+ resources => { optional => 0 },
+ nodes => { optional => 0 },
+ strict => { optional => 1 },
+ disable => { optional => 1 },
+ comment => { optional => 1 },
+ };
+}
+
+sub decode_plugin_value {
+ my ($class, $type, $key, $value) = @_;
+
+ if ($key eq 'nodes') {
+ my $res = {};
+
+ for my $node (PVE::Tools::split_list($value)) {
+ if (my ($node, $priority) = PVE::HA::Tools::parse_node_priority($node, 1)) {
+ $res->{$node} = {
+ priority => $priority,
+ };
+ }
+ }
+
+ return $res;
+ }
+
+ return $value;
+}
+
+sub encode_plugin_value {
+ my ($class, $type, $key, $value) = @_;
+
+ if ($key eq 'nodes') {
+ my $res = [];
+
+ for my $node (sort keys %$value) {
+ my $priority = $value->{$node}->{priority};
+
+ if ($priority) {
+ push @$res, "$node:$priority";
+ } else {
+ push @$res, "$node";
+ }
+ }
+
+ return join(',', @$res);
+ }
+
+ return $value;
+}
+
+sub get_plugin_check_arguments {
+ my ($self, $rules) = @_;
+
+ my $result = {
+ node_affinity_rules => {},
+ };
+
+ PVE::HA::Rules::foreach_rule(
+ $rules,
+ sub {
+ my ($rule, $ruleid) = @_;
+
+ $result->{node_affinity_rules}->{$ruleid} = $rule;
+ },
+ {
+ type => 'node-affinity',
+ exclude_disabled_rules => 1,
+ },
+ );
+
+ return $result;
+}
+
+=head1 NODE AFFINITY RULE CHECKERS
+
+=cut
+
+=head3 check_single_resource_reference($node_affinity_rules)
+
+Returns all in C<$node_affinity_rules> as a list of lists, each consisting of
+the node affinity id and the resource id, where at least one resource is shared
+between them.
+
+If there are none, the returned list is empty.
+
+=cut
+
+sub check_single_resource_reference {
+ my ($node_affinity_rules) = @_;
+
+ my @conflicts = ();
+ my $resource_ruleids = {};
+
+ while (my ($ruleid, $rule) = each %$node_affinity_rules) {
+ for my $sid (keys %{ $rule->{resources} }) {
+ push @{ $resource_ruleids->{$sid} }, $ruleid;
+ }
+ }
+
+ for my $sid (keys %$resource_ruleids) {
+ my $ruleids = $resource_ruleids->{$sid};
+
+ next if @$ruleids < 2;
+
+ for my $ruleid (@$ruleids) {
+ push @conflicts, [$ruleid, $sid];
+ }
+ }
+
+ @conflicts = sort { $a->[0] cmp $b->[0] } @conflicts;
+ return \@conflicts;
+}
+
+__PACKAGE__->register_check(
+ sub {
+ my ($args) = @_;
+
+ return check_single_resource_reference($args->{node_affinity_rules});
+ },
+ sub {
+ my ($conflicts, $errors) = @_;
+
+ for my $conflict (@$conflicts) {
+ my ($ruleid, $sid) = @$conflict;
+
+ push @{ $errors->{$ruleid}->{resources} },
+ "resource '$sid' is already used in another node affinity rule";
+ }
+ },
+);
+
+1;
diff --git a/src/PVE/HA/Tools.pm b/src/PVE/HA/Tools.pm
index 767659f..549cbe1 100644
--- a/src/PVE/HA/Tools.pm
+++ b/src/PVE/HA/Tools.pm
@@ -51,6 +51,18 @@ PVE::JSONSchema::register_standard_option(
},
);
+PVE::JSONSchema::register_standard_option(
+ 'pve-ha-resource-id-list',
+ {
+ description =>
+ "List of HA resource IDs. This consists of a list of resource types followed"
+ . " by a resource specific name separated with a colon (example: vm:100,ct:101).",
+ typetext => "<type>:<name>{,<type>:<name>}*",
+ type => 'string',
+ format => 'pve-ha-resource-id-list',
+ },
+);
+
PVE::JSONSchema::register_format('pve-ha-resource-or-vm-id', \&pve_verify_ha_resource_or_vm_id);
sub pve_verify_ha_resource_or_vm_id {
@@ -103,6 +115,18 @@ PVE::JSONSchema::register_standard_option(
},
);
+sub parse_node_priority {
+ my ($value, $noerr) = @_;
+
+ if ($value =~ m/^([a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?)(:(\d+))?$/) {
+ # node without priority set defaults to priority 0
+ return ($1, int($4 // 0));
+ }
+
+ return undef if $noerr;
+ die "unable to parse HA node entry '$value'\n";
+}
+
PVE::JSONSchema::register_standard_option(
'pve-ha-group-id',
{
--
2.39.5
More information about the pve-devel
mailing list