[pve-devel] [PATCH ha-manager 05/15] rules: add colocation rule plugin
Daniel Kral
d.kral at proxmox.com
Tue Mar 25 16:12:44 CET 2025
Add the colocation rule plugin to allow users to specify inter-service
affinity constraints.
These colocation rules can either be positive (keeping services
together) or negative (keeping service separate). Their strictness can
also be specified as either a MUST or a SHOULD, where the first
specifies that any service the constraint cannot be applied for stays in
recovery, while the latter specifies that that any service the
constraint cannot be applied for is lifted from the constraint.
The initial implementation also implements four basic transformations,
where colocation rules with not enough services are dropped, transitive
positive colocation rules are merged, and inter-colocation rule
inconsistencies as well as colocation rule inconsistencies with respect
to the location constraints specified in HA groups are dropped.
Signed-off-by: Daniel Kral <d.kral at proxmox.com>
---
debian/pve-ha-manager.install | 1 +
src/PVE/HA/Makefile | 1 +
src/PVE/HA/Rules/Colocation.pm | 391 +++++++++++++++++++++++++++++++++
src/PVE/HA/Rules/Makefile | 6 +
src/PVE/HA/Tools.pm | 6 +
5 files changed, 405 insertions(+)
create mode 100644 src/PVE/HA/Rules/Colocation.pm
create mode 100644 src/PVE/HA/Rules/Makefile
diff --git a/debian/pve-ha-manager.install b/debian/pve-ha-manager.install
index 9bbd375..89f9144 100644
--- a/debian/pve-ha-manager.install
+++ b/debian/pve-ha-manager.install
@@ -33,6 +33,7 @@
/usr/share/perl5/PVE/HA/Resources/PVECT.pm
/usr/share/perl5/PVE/HA/Resources/PVEVM.pm
/usr/share/perl5/PVE/HA/Rules.pm
+/usr/share/perl5/PVE/HA/Rules/Colocation.pm
/usr/share/perl5/PVE/HA/Tools.pm
/usr/share/perl5/PVE/HA/Usage.pm
/usr/share/perl5/PVE/HA/Usage/Basic.pm
diff --git a/src/PVE/HA/Makefile b/src/PVE/HA/Makefile
index 489cbc0..e386cbf 100644
--- a/src/PVE/HA/Makefile
+++ b/src/PVE/HA/Makefile
@@ -8,6 +8,7 @@ install:
install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA
for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/$$i; done
make -C Resources install
+ make -C Rules install
make -C Usage install
make -C Env install
diff --git a/src/PVE/HA/Rules/Colocation.pm b/src/PVE/HA/Rules/Colocation.pm
new file mode 100644
index 0000000..808d48e
--- /dev/null
+++ b/src/PVE/HA/Rules/Colocation.pm
@@ -0,0 +1,391 @@
+package PVE::HA::Rules::Colocation;
+
+use strict;
+use warnings;
+
+use Data::Dumper;
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::HA::Tools;
+
+use base qw(PVE::HA::Rules);
+
+sub type {
+ return 'colocation';
+}
+
+sub properties {
+ return {
+ services => get_standard_option('pve-ha-resource-id-list'),
+ affinity => {
+ description => "Describes whether the services are supposed to be kept on separate"
+ . " nodes, or are supposed to be kept together on the same node.",
+ type => 'string',
+ enum => ['separate', 'together'],
+ optional => 0,
+ },
+ strict => {
+ description => "Describes whether the colocation rule is mandatory or optional.",
+ type => 'boolean',
+ optional => 0,
+ },
+ }
+}
+
+sub options {
+ return {
+ services => { optional => 0 },
+ strict => { optional => 0 },
+ affinity => { optional => 0 },
+ comment => { optional => 1 },
+ };
+};
+
+sub decode_value {
+ my ($class, $type, $key, $value) = @_;
+
+ if ($key eq 'services') {
+ my $res = {};
+
+ for my $service (PVE::Tools::split_list($value)) {
+ if (PVE::HA::Tools::pve_verify_ha_resource_id($service)) {
+ $res->{$service} = 1;
+ }
+ }
+
+ return $res;
+ }
+
+ return $value;
+}
+
+sub encode_value {
+ my ($class, $type, $key, $value) = @_;
+
+ if ($key eq 'services') {
+ PVE::HA::Tools::pve_verify_ha_resource_id($_) for (keys %$value);
+
+ return join(',', keys %$value);
+ }
+
+ return $value;
+}
+
+sub foreach_colocation_rule {
+ my ($rules, $func, $opts) = @_;
+
+ my $my_opts = { map { $_ => $opts->{$_} } keys %$opts };
+ $my_opts->{type} = 'colocation';
+
+ PVE::HA::Rules::foreach_service_rule($rules, $func, $my_opts);
+}
+
+sub split_colocation_rules {
+ my ($rules) = @_;
+
+ my $positive_ruleids = [];
+ my $negative_ruleids = [];
+
+ foreach_colocation_rule($rules, sub {
+ my ($rule, $ruleid) = @_;
+
+ my $ruleid_set = $rule->{affinity} eq 'together' ? $positive_ruleids : $negative_ruleids;
+ push @$ruleid_set, $ruleid;
+ });
+
+ return ($positive_ruleids, $negative_ruleids);
+}
+
+=head3 check_service_count($rules)
+
+Returns a list of conflicts caused by colocation rules, which do not have
+enough services in them, defined in C<$rules>.
+
+If there are no conflicts, the returned list is empty.
+
+=cut
+
+sub check_services_count {
+ my ($rules) = @_;
+
+ my $conflicts = [];
+
+ foreach_colocation_rule($rules, sub {
+ my ($rule, $ruleid) = @_;
+
+ push @$conflicts, $ruleid if (scalar(keys %{$rule->{services}}) < 2);
+ });
+
+ return $conflicts;
+}
+
+=head3 check_positive_intransitivity($rules)
+
+Returns a list of conflicts caused by transitive positive colocation rules
+defined in C<$rules>.
+
+Transitive positive colocation rules exist, if there are at least two positive
+colocation rules with the same strictness, which put at least the same two
+services in relation. This means, that these rules can be merged together.
+
+If there are no conflicts, the returned list is empty.
+
+=cut
+
+sub check_positive_intransitivity {
+ my ($rules) = @_;
+
+ my $conflicts = {};
+ my ($positive_ruleids) = split_colocation_rules($rules);
+
+ while (my $outerid = shift(@$positive_ruleids)) {
+ my $outer = $rules->{ids}->{$outerid};
+
+ for my $innerid (@$positive_ruleids) {
+ my $inner = $rules->{ids}->{$innerid};
+
+ next if $outerid eq $innerid;
+ next if $outer->{strict} != $inner->{strict};
+ next if PVE::HA::Tools::is_disjoint($outer->{services}, $inner->{services});
+
+ push @{$conflicts->{$outerid}}, $innerid;
+ }
+ }
+
+ return $conflicts;
+}
+
+=head3 check_inner_consistency($rules)
+
+Returns a list of conflicts caused by inconsistencies between positive and
+negative colocation rules defined in C<$rules>.
+
+Inner inconsistent colocation rules exist, if there are at least the same two
+services in a positive and a negative colocation relation, which is an
+impossible constraint as they are opposites of each other.
+
+If there are no conflicts, the returned list is empty.
+
+=cut
+
+sub check_inner_consistency {
+ my ($rules) = @_;
+
+ my $conflicts = [];
+ my ($positive_ruleids, $negative_ruleids) = split_colocation_rules($rules);
+
+ for my $outerid (@$positive_ruleids) {
+ my $outer = $rules->{ids}->{$outerid}->{services};
+
+ for my $innerid (@$negative_ruleids) {
+ my $inner = $rules->{ids}->{$innerid}->{services};
+
+ my $intersection = PVE::HA::Tools::intersect($outer, $inner);
+ next if scalar(keys %$intersection < 2);
+
+ push @$conflicts, [$outerid, $innerid];
+ }
+ }
+
+ return $conflicts;
+}
+
+=head3 check_positive_group_consistency(...)
+
+Returns a list of conflicts caused by inconsistencies between positive
+colocation rules defined in C<$rules> and node restrictions defined in
+C<$groups> and C<$service>.
+
+A positive colocation rule inconsistency with groups exists, if at least two
+services in a positive colocation rule are restricted to disjoint sets of
+nodes, i.e. they are in restricted HA groups, which have a disjoint set of
+nodes.
+
+If there are no conflicts, the returned list is empty.
+
+=cut
+
+sub check_positive_group_consistency {
+ my ($rules, $groups, $services, $positive_ruleids, $conflicts) = @_;
+
+ for my $ruleid (@$positive_ruleids) {
+ my $rule_services = $rules->{ids}->{$ruleid}->{services};
+ my $nodes;
+
+ for my $sid (keys %$rule_services) {
+ my $groupid = $services->{$sid}->{group};
+ return if !$groupid;
+
+ my $group = $groups->{ids}->{$groupid};
+ return if !$group;
+ return if !$group->{restricted};
+
+ $nodes = { map { $_ => 1 } keys %{$group->{nodes}} } if !defined($nodes);
+ $nodes = PVE::HA::Tools::intersect($nodes, $group->{nodes});
+ }
+
+ if (defined($nodes) && scalar keys %$nodes < 1) {
+ push @$conflicts, ['positive', $ruleid];
+ }
+ }
+}
+
+=head3 check_negative_group_consistency(...)
+
+Returns a list of conflicts caused by inconsistencies between negative
+colocation rules defined in C<$rules> and node restrictions defined in
+C<$groups> and C<$service>.
+
+A negative colocation rule inconsistency with groups exists, if at least two
+services in a negative colocation rule are restricted to less nodes in total
+than services in the rule, i.e. they are in restricted HA groups, where the
+union of all restricted node sets have less elements than restricted services.
+
+If there are no conflicts, the returned list is empty.
+
+=cut
+
+sub check_negative_group_consistency {
+ my ($rules, $groups, $services, $negative_ruleids, $conflicts) = @_;
+
+ for my $ruleid (@$negative_ruleids) {
+ my $rule_services = $rules->{ids}->{$ruleid}->{services};
+ my $restricted_services = 0;
+ my $restricted_nodes;
+
+ for my $sid (keys %$rule_services) {
+ my $groupid = $services->{$sid}->{group};
+ return if !$groupid;
+
+ my $group = $groups->{ids}->{$groupid};
+ return if !$group;
+ return if !$group->{restricted};
+
+ $restricted_services++;
+
+ $restricted_nodes = {} if !defined($restricted_nodes);
+ $restricted_nodes = PVE::HA::Tools::union($restricted_nodes, $group->{nodes});
+ }
+
+ if (defined($restricted_nodes)
+ && scalar keys %$restricted_nodes < $restricted_services) {
+ push @$conflicts, ['negative', $ruleid];
+ }
+ }
+}
+
+sub check_consistency_with_groups {
+ my ($rules, $groups, $services) = @_;
+
+ my $conflicts = [];
+ my ($positive_ruleids, $negative_ruleids) = split_colocation_rules($rules);
+
+ check_positive_group_consistency($rules, $groups, $services, $positive_ruleids, $conflicts);
+ check_negative_group_consistency($rules, $groups, $services, $negative_ruleids, $conflicts);
+
+ return $conflicts;
+}
+
+sub canonicalize {
+ my ($class, $rules, $groups, $services) = @_;
+
+ my $illdefined_ruleids = check_services_count($rules);
+
+ for my $ruleid (@$illdefined_ruleids) {
+ print "Drop colocation rule '$ruleid', because it does not have enough services defined.\n";
+
+ delete $rules->{ids}->{$ruleid};
+ }
+
+ my $mergeable_positive_ruleids = check_positive_intransitivity($rules);
+
+ for my $outerid (sort keys %$mergeable_positive_ruleids) {
+ my $outer = $rules->{ids}->{$outerid};
+ my $innerids = $mergeable_positive_ruleids->{$outerid};
+
+ for my $innerid (@$innerids) {
+ my $inner = $rules->{ids}->{$innerid};
+
+ $outer->{services}->{$_} = 1 for (keys %{$inner->{services}});
+
+ print "Merge services of positive colocation rule '$innerid' into positive colocation"
+ . " rule '$outerid', because they share at least one service.\n";
+
+ delete $rules->{ids}->{$innerid};
+ }
+ }
+
+ my $inner_conflicts = check_inner_consistency($rules);
+
+ for my $conflict (@$inner_conflicts) {
+ my ($positiveid, $negativeid) = @$conflict;
+
+ print "Drop positive colocation rule '$positiveid' and negative colocation rule"
+ . " '$negativeid', because they share two or more services.\n";
+
+ delete $rules->{ids}->{$positiveid};
+ delete $rules->{ids}->{$negativeid};
+ }
+
+ my $group_conflicts = check_consistency_with_groups($rules, $groups, $services);
+
+ for my $conflict (@$group_conflicts) {
+ my ($type, $ruleid) = @$conflict;
+
+ if ($type eq 'positive') {
+ print "Drop positive colocation rule '$ruleid', because two or more services are"
+ . " restricted to different nodes.\n";
+ } elsif ($type eq 'negative') {
+ print "Drop negative colocation rule '$ruleid', because two or more services are"
+ . " restricted to less nodes than services.\n";
+ } else {
+ die "Invalid group conflict type $type\n";
+ }
+
+ delete $rules->{ids}->{$ruleid};
+ }
+}
+
+# TODO This will be used to verify modifications to the rules config over the API
+sub are_satisfiable {
+ my ($class, $rules, $groups, $services) = @_;
+
+ my $illdefined_ruleids = check_services_count($rules);
+
+ for my $ruleid (@$illdefined_ruleids) {
+ print "Colocation rule '$ruleid' does not have enough services defined.\n";
+ }
+
+ my $inner_conflicts = check_inner_consistency($rules);
+
+ for my $conflict (@$inner_conflicts) {
+ my ($positiveid, $negativeid) = @$conflict;
+
+ print "Positive colocation rule '$positiveid' is inconsistent with negative colocation rule"
+ . " '$negativeid', because they share two or more services between them.\n";
+ }
+
+ my $group_conflicts = check_consistency_with_groups($rules, $groups, $services);
+
+ for my $conflict (@$group_conflicts) {
+ my ($type, $ruleid) = @$conflict;
+
+ if ($type eq 'positive') {
+ print "Positive colocation rule '$ruleid' is unapplicable, because two or more services"
+ . " are restricted to different nodes.\n";
+ } elsif ($type eq 'negative') {
+ print "Negative colocation rule '$ruleid' is unapplicable, because two or more services"
+ . " are restricted to less nodes than services.\n";
+ } else {
+ die "Invalid group conflict type $type\n";
+ }
+ }
+
+ if (scalar(@$inner_conflicts) || scalar(@$group_conflicts)) {
+ return 0;
+ }
+
+ return 1;
+}
+
+1;
diff --git a/src/PVE/HA/Rules/Makefile b/src/PVE/HA/Rules/Makefile
new file mode 100644
index 0000000..8cb91ac
--- /dev/null
+++ b/src/PVE/HA/Rules/Makefile
@@ -0,0 +1,6 @@
+SOURCES=Colocation.pm
+
+.PHONY: install
+install:
+ install -d -m 0755 ${DESTDIR}${PERLDIR}/PVE/HA/Rules
+ for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/HA/Rules/$$i; done
diff --git a/src/PVE/HA/Tools.pm b/src/PVE/HA/Tools.pm
index 35107c9..52251d7 100644
--- a/src/PVE/HA/Tools.pm
+++ b/src/PVE/HA/Tools.pm
@@ -46,6 +46,12 @@ PVE::JSONSchema::register_standard_option('pve-ha-resource-id', {
type => 'string', format => 'pve-ha-resource-id',
});
+PVE::JSONSchema::register_standard_option('pve-ha-resource-id-list', {
+ description => "List of HA resource IDs.",
+ typetext => "<type>:<name>{,<type>:<name>}*",
+ type => 'string', format => 'pve-ha-resource-id-list',
+});
+
PVE::JSONSchema::register_format('pve-ha-resource-or-vm-id', \&pve_verify_ha_resource_or_vm_id);
sub pve_verify_ha_resource_or_vm_id {
my ($sid, $noerr) = @_;
--
2.39.5
More information about the pve-devel
mailing list