[pve-devel] [PATCH cluster] allow to add and delete qdevice to cluster

[Thomas Lamprecht]

Allow to add and delete qdevice through the pvecm CLI tool.

Needs to have corosync-qnetd installed on the host serving as
qdevice, it should be available for most current and popular linux
distributions. Further root SSH access from the PVE cluster to the
qdevice serving host is needed, at least during setup or deletion.
(only having public key auth (PermitRootLogin prohibit-password) is
naturally OK)

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---

I had this since quite a bit (> 1 year) laying around, I dusted it off a bit
(e.g., using subcommands) and did some simple add/delete tests

 data/PVE/CLI/pvecm.pm | 226 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 226 insertions(+)

diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
index 55c3f15..bb97c32 100755
--- a/data/PVE/CLI/pvecm.pm
+++ b/data/PVE/CLI/pvecm.pm
@@ -63,6 +63,228 @@ __PACKAGE__->register_method ({
 	return undef;
     }});
 
+my $foreach_member = sub {
+    my ($code, $noerr) = @_;
+
+    my $members = PVE::Cluster::get_members();
+    foreach my $node (sort keys %$members) {
+	if (my $ip = $members->{$node}->{ip}) {
+	    $code->($node, $ip);
+	} else {
+	    die "cannot get the cluster IP for node '$node'.\n" if !$noerr;
+	    warn "cannot get the cluster IP for node '$node'.\n";
+	    return undef;
+	}
+    }
+};
+
+__PACKAGE__->register_method ({
+    name => 'setup_qdevice',
+    path => 'setup_qdevice',
+    method => 'PUT',
+    description => "Setup the use of a QDevice",
+    parameters => {
+        additionalProperties => 0,
+	properties => {
+	    address => {
+		type => 'string', format => 'ip',
+		description => "Specifies the network address of an external corosync QDevice" ,
+	    },
+	    network => {
+		type => 'string',
+		format => 'CIDR',
+		description => 'The network which should be used to connect to the external qdevice',
+		optional => 1,
+	    },
+	    force => {
+		type => 'boolean',
+		description => "Do not throw error on possible dangerous operations.",
+		optional => 1,
+	    },
+	},
+    },
+    returns => { type => 'null' },
+
+    code => sub {
+	my ($param) = @_;
+
+	PVE::Corosync::check_conf_exists(1);
+
+	if (!PVE::Cluster::check_cfs_quorum(1)) {
+	    print "cluster must be quorate\n";
+	    return undef;
+	}
+
+	my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+
+	die "QDevice already configured!\n"
+	    if defined($conf->{main}->{quorum}->{device}) && !$param->{force};
+
+	my $network = $param->{network};
+
+	my $members = PVE::Cluster::get_members();
+
+	my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes' ];
+	my $base_cmd = ['pvecm', 'mtunnel', '--get_migration_ip', '--migration_network', $network];
+
+	# connect to all member to get their local IP on the given network
+	my $addresses = [];
+	foreach my $node (sort keys %$members) {
+	    die "All nodes must be online! Node '$node' is offline, abort.\n"
+		if !$members->{$node}->{online};
+
+	    if (my $ip = $members->{$node}->{ip}) {
+		my $nodeip = $ip;
+
+		if (defined($network)) {
+		    run_command([@$ssh_cmd, $ip, '--', @$base_cmd], outfunc => sub {
+			my $line = shift;
+
+			if ($line =~ m/^ip: '($PVE::Tools::IPRE)'$/) {
+			    $nodeip = $1;
+			} else {
+			    die "no ip found for node '$node' in network '$network'\n";
+			}
+		    }, errfunc => sub {
+			my $line = shift;
+
+			$line =~ s/^could not get migration ip: //;
+			die "error [$node]: $line\n";
+		    });
+
+		    print "found IP '$nodeip' on node '$node'\n";
+		}
+
+		push @$addresses, $nodeip;
+	    } else {
+		die "cannot get the cluster IP for node '$node', abort.\n";
+	    }
+	}
+
+	my $dev_address = $param->{address};
+
+	run_command(['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$dev_address"]);
+
+	if (-d "/etc/corosync/qdevice/net/nssdb") {
+	    # FIXME: check on all nodes?!
+	    if ($param->{force}) {
+		rmtree "/etc/corosync/qdevice/net/nssdb";
+	    } else {
+		die "QDevice certificate store already initialised, set force to delete!\n";
+	    }
+	}
+	print "Setup certificates for secure connection\n";
+	my $clustername = $conf->{main}->{totem}->{cluster_name};
+	my $cmd = ['corosync-qdevice-net-certutil', '-Q', '-n', $clustername, $dev_address, @$addresses];
+	run_command($cmd);
+
+
+	my $model = "net";
+	my $algorithm = 'ffsplit';
+	if (scalar($members) & 1) {
+	    if ($param->{force}) {
+		$algorithm = 'lms';
+	    } else {
+		die "Clusters with an odd node count are not officially supported!\n";
+	    }
+	}
+
+	my $code = sub {
+	    my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+	    my $quorum_section = $conf->{main}->{quorum};
+
+	    die "Qdevice already configured, must be deleted before setting up new one!\n"
+		if defined($quorum_section->{device}); # must not be forced!
+
+	    my $qdev_section = {
+		model => $model,
+		"$model" => {
+		    tls => 'on',
+		    host => $dev_address,
+		    algorithm => $algorithm,
+		}
+	    };
+	    $qdev_section->{votes} = 1 if $algorithm eq 'ffsplit';
+
+	    $quorum_section->{device} = $qdev_section;
+
+	    PVE::Corosync::atomic_write_conf($conf);
+	};
+
+	print "add QDevice to cluster configuration\n";
+	PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
+	die $@ if $@;
+
+	$foreach_member->(sub {
+	    my ($node, $ip) = @_;
+	    print "start and enable corosync qdevice daemon on node '$node'...\n";
+	    run_command(['ssh', $ip, 'systemctl', 'start', 'corosync-qdevice']);
+	    run_command(['ssh', $ip, 'systemctl', 'enable', 'corosync-qdevice']);
+	});
+
+	run_command(['corosync-cfgtool', '-R']); # do cluster wide config reload
+
+	print "done\n";
+
+	return undef;
+}});
+
+__PACKAGE__->register_method ({
+    name => 'delete_qdevice',
+    path => 'delete_qdevice',
+    method => 'DELETE',
+    description => "Remove a configured QDevice",
+    parameters => {
+        additionalProperties => 0,
+	properties => {},
+    },
+    returns => { type => 'null' },
+
+    code => sub {
+	my ($param) = @_;
+
+	PVE::Corosync::check_conf_exists(1);
+
+	if (!PVE::Cluster::check_cfs_quorum(1)) {
+	    # FIXME: *all* nodes must be online
+	    print "cluster must be quorate\n";
+	    return undef;
+	}
+
+	my $code = sub {
+	    my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+	    my $quorum_section = $conf->{main}->{quorum};
+
+	    die "No QDevice configured!\n" if !defined($quorum_section->{device});
+
+	    delete $quorum_section->{device};
+
+	    PVE::Corosync::atomic_write_conf($conf);
+
+	    # cleanup qdev state (cert storage)
+	    my $qdev_state_dir =  "/etc/corosync/qdevice";
+	    #rmtree $qdev_state_dir;
+
+	    $foreach_member->(sub {
+		my (undef, $ip) = @_;
+		run_command(['ssh', '-o', 'BatchMode=yes', $ip, '--', 'rm', '-rf', $qdev_state_dir ]);
+	    });
+	};
+
+	PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
+	die $@ if $@;
+
+	$foreach_member->(sub {
+	    my (undef, $ip) = @_;
+	    run_command(['ssh', $ip, 'systemctl', 'stop', 'corosync-qdevice']);
+	    run_command(['ssh', $ip, 'systemctl', 'disable', 'corosync-qdevice']);
+	});
+
+	run_command(['corosync-cfgtool', '-R']);
+
+	return undef;
+}});
+
 __PACKAGE__->register_method ({
     name => 'add',
     path => 'add',
@@ -396,6 +618,10 @@ our $cmddef = {
     expected => [ __PACKAGE__, 'expected', ['expected']],
     updatecerts => [ __PACKAGE__, 'updatecerts', []],
     mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
+    qdevice => {
+	setup => [ __PACKAGE__, 'setup_qdevice', ['address']],
+	delete => [ __PACKAGE__, 'delete_qdevice', []],
+    }
 };
 
 1;
-- 
2.18.0