[pve-devel] [PATCH cluster v2 7/7] WIP: add and delete qdevice

[Thomas Lamprecht]

Allow to add and delete qdevice through the pvecm CLI tool.

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---

NOTE: WIP

 data/PVE/CLI/pvecm.pm | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 208 insertions(+)

diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
index 5a93a66..ec71b72 100755
--- a/data/PVE/CLI/pvecm.pm
+++ b/data/PVE/CLI/pvecm.pm
@@ -98,6 +98,212 @@ __PACKAGE__->register_method ({
     }});
 
 __PACKAGE__->register_method ({
+    name => 'setup_qdevice',
+    path => 'setup_qdevice',
+    method => 'PUT',
+    description => "Setup the use of a QDevice",
+    parameters => {
+        additionalProperties => 0,
+	properties => {
+	    address => {
+		type => 'string', format => 'ip',
+		description => "Specifies the network address of an external corosync QDevice" ,
+	    },
+	    network => {
+		type => 'string',
+		format => 'CIDR',
+		description => 'The network which should be used to connect to the external qdevice',
+		optional => 1,
+	    },
+	    force => {
+		type => 'boolean',
+		description => "Do not throw error on possible dangerous operations.",
+		optional => 1,
+	    },
+	},
+    },
+    returns => { type => 'null' },
+
+    code => sub {
+	my ($param) = @_;
+
+	PVE::Corosync::check_conf_exists(1);
+
+	if (!PVE::Cluster::check_cfs_quorum(1)) {
+	    print "cluster must be quorate\n";
+	    return undef;
+	}
+
+	my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+
+	die "QDevice already configured!\n"
+	    if defined($conf->{main}->{quorum}->{device}) && !$param->{force};
+
+	my $network = $param->{network};
+
+	my $members = PVE::Cluster::get_members();
+
+	my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes' ];
+	my $base_cmd = ['pvecm', 'mtunnel', '--get_migration_ip', '--migration_network', $network];
+
+	# connect to all member to get their local IP on the given network
+	my $addresses = [];
+	foreach my $node (sort keys %$members) {
+	    die "All nodes must be online! Node '$node' is offline, abort.\n"
+		if !$members->{$node}->{online};
+
+	    if (my $ip = $members->{$node}->{ip}) {
+		my $nodeip = $ip;
+
+		if (defined($network)) {
+		    run_command([@$ssh_cmd, $ip, '--', @$base_cmd], outfunc => sub {
+			my $line = shift;
+
+			if ($line =~ m/^ip: '($PVE::Tools::IPRE)'$/) {
+			    $nodeip = $1;
+			} else {
+			    die "no ip found for node '$node' in network '$network'\n";
+			}
+		    }, errfunc => sub {
+			my $line = shift;
+
+			$line =~ s/^could not get migration ip: //;
+			die "error [$node]: $line\n";
+		    });
+
+		    print "found IP '$nodeip' on node '$node'\n";
+		}
+
+		push @$addresses, $nodeip;
+	    } else {
+		die "cannot get the cluster IP for node '$node', abort.\n";
+	    }
+	}
+
+	my $dev_address = $param->{address};
+
+	run_command(['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$dev_address"]);
+
+	if (-d "/etc/corosync/qdevice/net/nssdb") {
+	    # FIXME: check on all nodes?!
+	    if ($param->{force}) {
+		rmtree "/etc/corosync/qdevice/net/nssdb";
+	    } else {
+		die "QDevice certificate store already initialised, set force to delete!\n";
+	    }
+	}
+	print "Setup certificates for secure connection\n";
+	my $clustername = $conf->{main}->{totem}->{cluster_name};
+	my $cmd = ['corosync-qdevice-net-certutil', '-Q', '-n', $clustername, $dev_address, @$addresses];
+	run_command($cmd);
+
+
+	my $model = "net";
+	my $algorithm = 'ffsplit';
+	if (scalar($members) & 1) {
+	    if ($param->{force}) {
+		$algorithm = 'lms';
+	    } else {
+		die "Clusters with an odd node count are not officially supported!\n";
+	    }
+	}
+
+	my $code = sub {
+	    my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+	    my $quorum_section = $conf->{main}->{quorum};
+
+	    die "Qdevice already configured, must be deleted before setting up new one!\n"
+		if defined($quorum_section->{device}); # must not be forced!
+
+	    my $qdev_section = {
+		model => $model,
+		"$model" => {
+		    tls => 'on',
+		    host => $dev_address,
+		    algorithm => $algorithm,
+		}
+	    };
+	    $qdev_section->{votes} = 1 if $algorithm eq 'ffsplit';
+
+	    $quorum_section->{device} = $qdev_section;
+
+	    PVE::Corosync::atomic_write_conf($conf, 1);
+	};
+
+	print "add QDevice to cluster configuration\n";
+	PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
+	die $@ if $@;
+
+	PVE::Cluster::foreach_member(sub {
+	    my ($node, $ip) = @_;
+	    run_command(['ssh', $ip, 'systemctl', 'start', 'corosync-qdevice']);
+	    run_command(['ssh', $ip, 'systemctl', 'enable', 'corosync-qdevice']);
+	});
+
+	run_command(['corosync-cfgtool', '-R']); # do cluster wide config reload
+
+	print "done\n";
+
+	return undef;
+}});
+
+__PACKAGE__->register_method ({
+    name => 'delete_qdevice',
+    path => 'delete_qdevice',
+    method => 'DELETE',
+    description => "Remove a configured QDevice",
+    parameters => {
+        additionalProperties => 0,
+	properties => {},
+    },
+    returns => { type => 'null' },
+
+    code => sub {
+	my ($param) = @_;
+
+	PVE::Corosync::check_conf_exists(1);
+
+	if (!PVE::Cluster::check_cfs_quorum(1)) {
+	    # FIXME: *all* nodes must be online
+	    print "cluster must be quorate\n";
+	    return undef;
+	}
+
+	my $code = sub {
+	    my $conf = PVE::Cluster::cfs_read_file("corosync.conf");
+	    my $quorum_section = $conf->{main}->{quorum};
+
+	    die "No QDevice configured!\n" if !defined($quorum_section->{device});
+
+	    delete $quorum_section->{device};
+
+	    PVE::Corosync::atomic_write_conf($conf, 1);
+
+	    # cleanup qdev state (cert storage)
+	    my $qdev_state_dir =  "/etc/corosync/qdevice";
+	    #rmtree $qdev_state_dir;
+
+	    PVE::Cluster::foreach_member(sub {
+		my (undef, $ip) = @_;
+		run_command(['ssh', '-o', 'BatchMode=yes', $ip, '--', 'rm', '-rf', $qdev_state_dir ]);
+	    });
+	};
+
+	PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code);
+	die $@ if $@;
+
+	PVE::Cluster::foreach_member(sub {
+	    my (undef, $ip) = @_;
+	    run_command(['ssh', $ip, 'systemctl', 'stop', 'corosync-qdevice']);
+	    run_command(['ssh', $ip, 'systemctl', 'disable', 'corosync-qdevice']);
+	});
+
+	run_command(['corosync-cfgtool', '-R']);
+
+	return undef;
+}});
+
+__PACKAGE__->register_method ({
     name => 'create',
     path => 'create',
     method => 'PUT',
@@ -888,6 +1094,8 @@ our $cmddef = {
     expected => [ __PACKAGE__, 'expected', ['expected']],
     updatecerts => [ __PACKAGE__, 'updatecerts', []],
     mtunnel => [ __PACKAGE__, 'mtunnel', ['extra-args']],
+    setup_qdevice => [ __PACKAGE__, 'setup_qdevice', ['address']],
+    delete_qdevice => [ __PACKAGE__, 'delete_qdevice', []],
 };
 
 1;
-- 
2.11.0