[pve-devel] [PATCH v2 cluster 2/5] Enable support for up to 8 corosync links
Thomas Lamprecht
t.lamprecht at proxmox.com
Tue Dec 17 06:38:01 CET 2019
On 12/4/19 3:04 PM, Stefan Reiter wrote:
> add_corosync_link_properties/extract_corosync_link_args are introduced
> as helpers to avoid hardcoding links in parameters=>properties on
> several occasions, while still providing autocompletion with pvecm by
> being seperate parameters instead of an array.
>
> Maximum number of links is given as constant MAX_LINK_COUNT, should it
> change in the future.
>
> All necessary functions have been updated to
> use the new $links array format instead of seperate $link0/$link1
> parameters, and call sites changed accordingly.
>
> Signed-off-by: Stefan Reiter <s.reiter at proxmox.com>
> ---
>
> This patch intentionally removes some verification and fallback code to be
> reintroduced in the next one.
not really tested, but doesn't smells after a semi-close look ;)
>
> data/PVE/API2/ClusterConfig.pm | 51 +++++++-------------
> data/PVE/CLI/pvecm.pm | 21 ++++----
> data/PVE/Cluster/Setup.pm | 22 +++++----
> data/PVE/Corosync.pm | 88 +++++++++++++++++++++++-----------
> 4 files changed, 99 insertions(+), 83 deletions(-)
>
> diff --git a/data/PVE/API2/ClusterConfig.pm b/data/PVE/API2/ClusterConfig.pm
> index e05fd55..2c5bb03 100644
> --- a/data/PVE/API2/ClusterConfig.pm
> +++ b/data/PVE/API2/ClusterConfig.pm
> @@ -68,10 +68,11 @@ __PACKAGE__->register_method ({
> path => '',
> method => 'POST',
> protected => 1,
> - description => "Generate new cluster configuration.",
> + description => "Generate new cluster configuration. If no links given,"
> + . " default to local IP address as link0.",
> parameters => {
> additionalProperties => 0,
> - properties => {
> + properties => PVE::Corosync::add_corosync_link_properties({
> clustername => {
> description => "The name of the cluster.",
> type => 'string', format => 'pve-node',
> @@ -84,9 +85,7 @@ __PACKAGE__->register_method ({
> minimum => 1,
> optional => 1,
> },
> - link0 => get_standard_option('corosync-link'),
> - link1 => get_standard_option('corosync-link'),
> - },
> + }),
> },
> returns => { type => 'string' },
> code => sub {
> @@ -110,7 +109,7 @@ __PACKAGE__->register_method ({
> my $nodename = PVE::INotify::nodename();
>
> # get the corosync basis config for the new cluster
> - my $config = PVE::Corosync::create_conf($nodename, %$param);
> + my $config = PVE::Corosync::create_conf($nodename, $param);
>
> print "Writing corosync config to /etc/pve/corosync.conf\n";
> PVE::Corosync::atomic_write_conf($config);
> @@ -194,7 +193,7 @@ __PACKAGE__->register_method ({
> description => "Adds a node to the cluster configuration. This call is for internal use.",
> parameters => {
> additionalProperties => 0,
> - properties => {
> + properties => PVE::Corosync::add_corosync_link_properties({
> node => get_standard_option('pve-node'),
> nodeid => get_standard_option('corosync-nodeid'),
> votes => {
> @@ -208,9 +207,7 @@ __PACKAGE__->register_method ({
> description => "Do not throw error if node already exists.",
> optional => 1,
> },
> - link0 => get_standard_option('corosync-link'),
> - link1 => get_standard_option('corosync-link'),
> - },
> + }),
> },
> returns => {
> type => "object",
> @@ -256,7 +253,7 @@ __PACKAGE__->register_method ({
> while (my ($k, $v) = each %$nodelist) {
> next if $k eq $name; # allows re-adding a node if force is set
>
> - for my $linknumber (0..1) {
> + for my $linknumber (0..PVE::Corosync::MAX_LINK_INDEX) {
> my $id = "ring${linknumber}_addr";
> next if !defined($v->{$id});
>
> @@ -266,20 +263,10 @@ __PACKAGE__->register_method ({
> }
> };
>
> - my $link0 = PVE::Corosync::parse_corosync_link($param->{link0});
> - my $link1 = PVE::Corosync::parse_corosync_link($param->{link1});
> -
> - $check_duplicate_addr->($link0);
> - $check_duplicate_addr->($link1);
> -
> - # FIXME: handle all links (0-7), they're all independent now
> - $link0->{address} //= $name if exists($totem_cfg->{interface}->{0});
> -
> - die "corosync: using 'link1' parameter needs a interface with linknumber '1' configured!\n"
> - if $link1 && !defined($totem_cfg->{interface}->{1});
> -
> - die "corosync: totem interface with linknumber 1 configured but 'link1' parameter not defined!\n"
> - if defined($totem_cfg->{interface}->{1}) && !defined($link1);
> + my $links = PVE::Corosync::extract_corosync_link_args($param);
> + foreach my $link (values %$links) {
> + $check_duplicate_addr->($link);
> + }
>
> if (defined(my $res = $nodelist->{$name})) {
> $param->{nodeid} = $res->{nodeid} if !$param->{nodeid};
> @@ -317,13 +304,15 @@ __PACKAGE__->register_method ({
> warn $@ if $@;
>
> $nodelist->{$name} = {
> - ring0_addr => $link0->{address},
> nodeid => $param->{nodeid},
> name => $name,
> };
> - $nodelist->{$name}->{ring1_addr} = $link1->{address} if defined($link1);
> $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes};
>
> + foreach my $link (keys %$links) {
> + $nodelist->{$name}->{"ring${link}_addr"} = $links->{$link}->{address};
> + }
> +
> PVE::Cluster::log_msg('notice', 'root at pam', "adding node $name to cluster");
>
> PVE::Corosync::update_nodelist($conf, $nodelist);
> @@ -508,7 +497,7 @@ __PACKAGE__->register_method ({
> description => "Joins this node into an existing cluster.",
> parameters => {
> additionalProperties => 0,
> - properties => {
> + properties => PVE::Corosync::add_corosync_link_properties({
> hostname => {
> type => 'string',
> description => "Hostname (or IP) of an existing cluster member."
> @@ -525,17 +514,13 @@ __PACKAGE__->register_method ({
> description => "Do not throw error if node already exists.",
> optional => 1,
> },
> - link0 => get_standard_option('corosync-link', {
> - default => "IP resolved by node's hostname",
> - }),
> - link1 => get_standard_option('corosync-link'),
> fingerprint => get_standard_option('fingerprint-sha256'),
> password => {
> description => "Superuser (root) password of peer node.",
> type => 'string',
> maxLength => 128,
> },
> - },
> + }),
> },
> returns => { type => 'string' },
> code => sub {
> diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
> index c634c9b..df0b1b9 100755
> --- a/data/PVE/CLI/pvecm.pm
> +++ b/data/PVE/CLI/pvecm.pm
> @@ -316,7 +316,7 @@ __PACKAGE__->register_method ({
> description => "Adds the current node to an existing cluster.",
> parameters => {
> additionalProperties => 0,
> - properties => {
> + properties => PVE::Corosync::add_corosync_link_properties({
> hostname => {
> type => 'string',
> description => "Hostname (or IP) of an existing cluster member."
> @@ -333,8 +333,6 @@ __PACKAGE__->register_method ({
> description => "Do not throw error if node already exists.",
> optional => 1,
> },
> - link0 => get_standard_option('corosync-link'),
> - link1 => get_standard_option('corosync-link'),
> fingerprint => get_standard_option('fingerprint-sha256', {
> optional => 1,
> }),
> @@ -343,7 +341,7 @@ __PACKAGE__->register_method ({
> description => "Always use SSH to join, even if peer may do it over API.",
> optional => 1,
> },
> - },
> + }),
> },
> returns => { type => 'null' },
>
> @@ -377,11 +375,9 @@ __PACKAGE__->register_method ({
> # allow fallback to old ssh only join if wished or needed
>
> my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
> + my $links = PVE::Corosync::extract_corosync_link_args($param);
>
> - my $link0 = PVE::Corosync::parse_corosync_link($param->{link0});
> - my $link1 = PVE::Corosync::parse_corosync_link($param->{link1});
> -
> - PVE::Cluster::Setup::assert_joinable($local_ip_address, $link0, $link1, $param->{force});
> + PVE::Cluster::Setup::assert_joinable($local_ip_address, $links, $param->{force});
>
> PVE::Cluster::Setup::setup_sshd_config();
> PVE::Cluster::Setup::setup_rootsshconfig();
> @@ -399,10 +395,11 @@ __PACKAGE__->register_method ({
>
> push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
> push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
> - # just pass the un-parsed string through, or as we've address as
> - # the default_key, we can just pass the fallback directly too
> - push @$cmd, '--link0', $param->{link0} // $local_ip_address;
> - push @$cmd, '--link1', $param->{link1} if defined($param->{link1});
> +
> + foreach my $link (keys %$links) {
> + push @$cmd, "--link$link", PVE::JSONSchema::print_property_string(
> + $links->{$link}, get_standard_option('corosync-link'));
> + }
>
> if (system (@$cmd) != 0) {
> my $cmdtxt = join (' ', @$cmd);
> diff --git a/data/PVE/Cluster/Setup.pm b/data/PVE/Cluster/Setup.pm
> index b115d45..71a637f 100644
> --- a/data/PVE/Cluster/Setup.pm
> +++ b/data/PVE/Cluster/Setup.pm
> @@ -579,7 +579,7 @@ sub gen_pve_vzdump_files {
> # join helpers
>
> sub assert_joinable {
> - my ($local_addr, $link0, $link1, $force) = @_;
> + my ($local_addr, $links, $force) = @_;
>
> my $errors = '';
> my $error = sub { $errors .= "* $_[0]\n"; };
> @@ -622,8 +622,12 @@ sub assert_joinable {
> };
>
> $check_ip->($local_addr, 'local node address');
> - $check_ip->($link0->{address}, 'ring0') if defined($link0);
> - $check_ip->($link1->{address}, 'ring1') if defined($link1);
> +
> + if ($links) {
> + foreach my $link (keys %$links) {
> + $check_ip->($links->{$link}->{address}, "link$link");
> + }
> + }
>
> if ($errors) {
> warn "detected the following error(s):\n$errors";
> @@ -637,11 +641,10 @@ sub join {
> my $nodename = PVE::INotify::nodename();
> my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
>
> - my $link0 = PVE::Corosync::parse_corosync_link($param->{link0});
> - my $link1 = PVE::Corosync::parse_corosync_link($param->{link1});
> + my $links = PVE::Corosync::extract_corosync_link_args($param);
>
> # check if we can join with the given parameters and current node state
> - assert_joinable($local_ip_address, $link0, $link1, $param->{force});
> + assert_joinable($local_ip_address, $links, $param->{force});
>
> setup_sshd_config();
> setup_rootsshconfig();
> @@ -679,10 +682,9 @@ sub join {
> $args->{force} = $param->{force} if defined($param->{force});
> $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
> $args->{votes} = $param->{votes} if defined($param->{votes});
> - # just pass the un-parsed string through, or as we've address as the
> - # default_key, we can just pass the fallback directly too
> - $args->{link0} = $param->{link0} // $local_ip_address;
> - $args->{link1} = $param->{link1} if defined($param->{link1});
> + foreach my $link (keys %$links) {
> + $args->{"link$link"} = PVE::Corosync::print_corosync_link($links->{$link});
> + }
>
> print "Request addition of this node\n";
> my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
> diff --git a/data/PVE/Corosync.pm b/data/PVE/Corosync.pm
> index d282aed..caec799 100644
> --- a/data/PVE/Corosync.pm
> +++ b/data/PVE/Corosync.pm
> @@ -35,12 +35,15 @@ my $corosync_link_format = {
> minimum => 0,
> maximum => 255,
> default => 0,
> - description => "The priority for the link when knet is used in 'passive' mode. Lower value means higher priority.",
> + description => "The priority for the link when knet is used in 'passive'"
> + . " mode (default). Lower value means higher priority. Only"
> + . " valid for cluster create, ignored on node add.",
> },
> };
> my $corosync_link_desc = {
> type => 'string', format => $corosync_link_format,
> - description => "Address and priority information of a single corosync link.",
> + description => "Address and priority information of a single corosync link."
> + . " (up to 8 links supported; link0..link7)",
> optional => 1,
> };
> PVE::JSONSchema::register_standard_option("corosync-link", $corosync_link_desc);
> @@ -53,6 +56,38 @@ sub parse_corosync_link {
> return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
> }
>
> +sub print_corosync_link {
> + my ($link) = @_;
> +
> + return undef if !defined($link);
> +
> + return PVE::JSONSchema::print_property_string($link, $corosync_link_format);
> +}
> +
> +use constant MAX_LINK_INDEX => 7;
> +
> +sub add_corosync_link_properties {
> + my ($prop) = @_;
> +
> + for my $lnum (0..MAX_LINK_INDEX) {
> + $prop->{"link$lnum"} = PVE::JSONSchema::get_standard_option("corosync-link");
> + }
> +
> + return $prop;
> +}
> +
> +sub extract_corosync_link_args {
> + my ($args) = @_;
> +
> + my $links = {};
> + for my $lnum (0..MAX_LINK_INDEX) {
> + $links->{$lnum} = parse_corosync_link($args->{"link$lnum"})
> + if $args->{"link$lnum"};
> + }
> +
> + return $links;
> +}
> +
> # a very simply parser ...
> sub parse_conf {
> my ($filename, $raw) = @_;
> @@ -232,16 +267,19 @@ sub atomic_write_conf {
> # for creating a new cluster with the current node
> # params are those from the API/CLI cluster create call
> sub create_conf {
> - my ($nodename, %param) = @_;
> + my ($nodename, $param) = @_;
>
> - my $clustername = $param{clustername};
> - my $nodeid = $param{nodeid} || 1;
> - my $votes = $param{votes} || 1;
> + my $clustername = $param->{clustername};
> + my $nodeid = $param->{nodeid} || 1;
> + my $votes = $param->{votes} || 1;
>
> my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
>
> - my $link0 = parse_corosync_link($param{link0});
> - $link0->{address} //= $local_ip_address;
> + my $links = extract_corosync_link_args($param);
> +
> + # if no links given, fall back to local IP as link0
> + $links->{0} = { address => $local_ip_address }
> + if !%$links;
>
> my $conf = {
> totem => {
> @@ -250,11 +288,8 @@ sub create_conf {
> cluster_name => $clustername,
> config_version => 0,
> ip_version => 'ipv4-6',
> - interface => {
> - 0 => {
> - linknumber => 0,
> - },
> - },
> + link_mode => 'passive',
> + interface => {},
> },
> nodelist => {
> node => {
> @@ -262,7 +297,6 @@ sub create_conf {
> name => $nodename,
> nodeid => $nodeid,
> quorum_votes => $votes,
> - ring0_addr => $link0->{address},
> },
> },
> },
> @@ -275,19 +309,17 @@ sub create_conf {
> },
> };
> my $totem = $conf->{totem};
> + my $node = $conf->{nodelist}->{node}->{$nodename};
> +
> + foreach my $lnum (keys %$links) {
> + my $link = $links->{$lnum};
> +
> + $totem->{interface}->{$lnum} = { linknumber => $lnum };
> +
> + my $prio = $link->{priority};
> + $totem->{interface}->{$lnum}->{knet_link_priority} = $prio if $prio;
>
> - $totem->{interface}->{0}->{knet_link_priority} = $link0->{priority}
> - if defined($link0->{priority});
> -
> - my $link1 = parse_corosync_link($param{link1});
> - if ($link1->{address}) {
> - $conf->{totem}->{interface}->{1} = {
> - linknumber => 1,
> - };
> - $totem->{link_mode} = 'passive';
> - $totem->{interface}->{1}->{knet_link_priority} = $link1->{priority}
> - if defined($link1->{priority});
> - $conf->{nodelist}->{node}->{$nodename}->{ring1_addr} = $link1->{address};
> + $node->{"ring${lnum}_addr"} = $link->{address};
> }
>
> return { main => $conf };
> @@ -353,7 +385,7 @@ sub verify_conf {
> if (%$interfaces) {
> # if interfaces are defined, *all* links must have a matching interface
> # definition, and vice versa
> - for my $link (0..1) {
> + for my $link (0..MAX_LINK_INDEX) {
> my $have_interface = defined($interfaces->{$link});
> foreach my $node (@node_names) {
> my $linkdef = $node_links->{$node}->{$link};
> @@ -371,7 +403,7 @@ sub verify_conf {
> }
> } else {
> # without interfaces, only check that links are consistent among nodes
> - for my $link (0..1) {
> + for my $link (0..MAX_LINK_INDEX) {
> my $nodes_with_link = {};
> foreach my $node (@node_names) {
> my $linkdef = $node_links->{$node}->{$link};
>
More information about the pve-devel
mailing list