[pve-devel] [PATCH v3 cluster 4/4] Add cluster join API version check
Stefan Reiter
s.reiter at proxmox.com
Thu Jan 9 16:31:36 CET 2020
Adds API call GET /cluster/config/apiversion to retrieve remote clusters
join-API version (0 is assumed for versions without this endpoint). Also
available via CLI as 'pvecm apiver'.
Introduce API_AGE similar to storage plugin API, but with two ages for
cluster/joinee roles. Currently, all versions are intercompatible.
For future usage, a new 'addnode' parameter 'apiversion' is introduced,
to allow introducing API breakages for joining nodes as well.
As a first compatibility check, use new fallback method only if
available. This ensures full compatibility between nodes/clusters with
and without new fallback behaviour.
Signed-off-by: Stefan Reiter <s.reiter at proxmox.com>
---
I do think that both AS_CLUSTER and AS_JOINEE ages are necessary indeed:
* AS_CLUSTER: If we introduce a parameter to addnode that we cannot have a
fallback for, and old versions don't provide it, we can exclude them from
joining
* AS_JOINEE: If something changes regarding the cluster joing procedure and we
know that older versions cannot correctly add us to the cluster, we can abort
the join
If I'm wrong, it'd be easy enough to remove one or the other. Having both
doesn't hurt though?
data/PVE/API2/ClusterConfig.pm | 37 +++++++++++++++++++++++++++++++++-
data/PVE/CLI/pvecm.pm | 22 +++++++++++++++++++-
data/PVE/Cluster/Setup.pm | 23 ++++++++++++++++++++-
3 files changed, 79 insertions(+), 3 deletions(-)
diff --git a/data/PVE/API2/ClusterConfig.pm b/data/PVE/API2/ClusterConfig.pm
index e6b59cf..205252f 100644
--- a/data/PVE/API2/ClusterConfig.pm
+++ b/data/PVE/API2/ClusterConfig.pm
@@ -58,11 +58,33 @@ __PACKAGE__->register_method({
{ name => 'totem' },
{ name => 'join' },
{ name => 'qdevice' },
+ { name => 'apiversion' },
];
return $result;
}});
+__PACKAGE__->register_method ({
+ name => 'join_api_version',
+ path => 'apiversion',
+ method => 'GET',
+ description => "Return the version of the cluster join API available on this node.",
+ permissions => {
+ check => ['perm', '/', [ 'Sys.Audit' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {},
+ },
+ returns => {
+ type => 'integer',
+ minimum => 0,
+ description => "Cluster Join API version, currently " . PVE::Cluster::Setup::JOIN_API_VERSION,
+ },
+ code => sub {
+ return PVE::Cluster::Setup::JOIN_API_VERSION;
+ }});
+
__PACKAGE__->register_method ({
name => 'create',
path => '',
@@ -213,6 +235,11 @@ __PACKAGE__->register_method ({
format => 'ip',
optional => 1,
},
+ apiversion => {
+ type => 'integer',
+ description => 'The JOIN_API_VERSION of the new node.',
+ optional => 1,
+ },
}),
},
returns => {
@@ -235,6 +262,14 @@ __PACKAGE__->register_method ({
code => sub {
my ($param) = @_;
+ $param->{apiversion} //= 0;
+ if ($param->{apiversion} < (PVE::Cluster::Setup::JOIN_API_VERSION -
+ PVE::Cluster::Setup::JOIN_API_AGE_AS_CLUSTER)) {
+ die "unsupported old API version on joining node ($param->{apiversion},"
+ . " cluster node has " . PVE::Cluster::Setup::JOIN_API_VERSION
+ . "), please upgrade before joining\n";
+ }
+
PVE::Cluster::check_cfs_quorum();
my $vc_errors;
@@ -291,7 +326,7 @@ __PACKAGE__->register_method ({
# FIXME: remove in 8.0 or when joining an old node not supporting
# new_node_ip becomes infeasible otherwise
my $legacy_fallback = 0;
- if (!$param->{new_node_ip} && scalar(%$links) == 1) {
+ if (!$param->{new_node_ip} && scalar(%$links) == 1 && $param->{apiversion} == 0) {
my $passed_link_id = (keys %$links)[0];
my $passed_link = delete $links->{$passed_link_id};
$param->{new_node_ip} = $passed_link->{address};
diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
index a838ff2..3ce80a6 100755
--- a/data/PVE/CLI/pvecm.pm
+++ b/data/PVE/CLI/pvecm.pm
@@ -390,6 +390,20 @@ __PACKAGE__->register_method ({
run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
'errmsg' => "unable to copy ssh ID");
+ $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'apiver'];
+ my $remote_apiver = 0;
+ run_command($cmd, 'outfunc' => sub {
+ $remote_apiver = shift;
+ chomp $remote_apiver;
+ }, 'noerr' => 1);
+
+ if ($remote_apiver < (PVE::Cluster::Setup::JOIN_API_VERSION -
+ PVE::Cluster::Setup::JOIN_API_AGE_AS_JOINEE)) {
+ die "error: incompatible join API version on cluster ($remote_apiver,"
+ . " local has " . PVE::Cluster::Setup::JOIN_API_VERSION . "). Make"
+ . " sure all nodes are up-to-date.\n";
+ }
+
$cmd = ['ssh', $host, '-o', 'BatchMode=yes',
'pvecm', 'addnode', $nodename, '--force', 1];
@@ -403,7 +417,9 @@ __PACKAGE__->register_method ({
# this will be used as fallback if no links are specified
if (!%$links) {
- push @$cmd, '--link0', $local_ip_address;
+ push @$cmd, '--link0', $local_ip_address if $remote_apiver == 0;
+ push @$cmd, '--new_node_ip', $local_ip_address if $remote_apiver >= 1;
+
print "No cluster network links passed explicitly, fallback to local node"
. " IP '$local_ip_address'\n";
}
@@ -684,6 +700,10 @@ __PACKAGE__->register_method ({
our $cmddef = {
+ apiver => [ 'PVE::API2::ClusterConfig', 'join_api_version', [], {}, sub {
+ my $apiver = shift;
+ print "$apiver\n";
+ }],
keygen => [ __PACKAGE__, 'keygen', ['filename']],
create => [ 'PVE::API2::ClusterConfig', 'create', ['clustername']],
add => [ __PACKAGE__, 'add', ['hostname']],
diff --git a/data/PVE/Cluster/Setup.pm b/data/PVE/Cluster/Setup.pm
index 27e2403..2ffb8f9 100644
--- a/data/PVE/Cluster/Setup.pm
+++ b/data/PVE/Cluster/Setup.pm
@@ -20,6 +20,13 @@ use PVE::Network;
use PVE::Tools;
use PVE::Certificate;
+# Only relevant for pre-join checks, after join happened versions can differ
+use constant JOIN_API_VERSION => 1;
+# (APIVER-this) is oldest version a new node in addnode can have to be accepted
+use constant JOIN_API_AGE_AS_CLUSTER => 1;
+# (APIVER-this) is oldest version a cluster node can have to still try joining
+use constant JOIN_API_AGE_AS_JOINEE => 1;
+
my $pmxcfs_base_dir = PVE::Cluster::base_dir();
my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
@@ -676,6 +683,14 @@ sub join {
# login raises an exception on failure, so if we get here we're good
print "Login succeeded.\n";
+ # check cluster join API version
+ my $apiver = eval { $conn->get("/cluster/config/apiversion") } // 0;
+
+ if ($apiver < (JOIN_API_VERSION - JOIN_API_AGE_AS_JOINEE)) {
+ die "error: incompatible join API version on cluster ($apiver, local has "
+ . JOIN_API_VERSION . "). Make sure all nodes are up-to-date.\n";
+ }
+
my $args = {};
$args->{force} = $param->{force} if defined($param->{force});
$args->{nodeid} = $param->{nodeid} if $param->{nodeid};
@@ -686,11 +701,17 @@ sub join {
# this will be used as fallback if no links are specified
if (!%$links) {
- $args->{link0} = $local_ip_address;
+ $args->{link0} = $local_ip_address if $apiver == 0;
+ $args->{new_node_ip} = $local_ip_address if $apiver >= 1;
+
print "No cluster network links passed explicitly, fallback to local node"
. " IP '$local_ip_address'\n";
}
+ if ($apiver >= 1) {
+ $args->{apiversion} = JOIN_API_VERSION;
+ }
+
print "Request addition of this node\n";
my $res = eval { $conn->post("/cluster/config/nodes/$nodename", $args); };
if (my $err = $@) {
--
2.20.1
More information about the pve-devel
mailing list