[pve-devel] [PATCH v2 cluster 03/13] move cluster setup related code to own file

Fabian Grünbichler f.gruenbichler at proxmox.com
Mon Nov 11 11:27:57 CET 2019


this includes:
- directory setup
- ssh setup (known hosts, keys, config)
- CA/certificate helpers
- join helpers used by API and CLI code

Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
 data/PVE/Cluster/Makefile      |   2 +-
 data/PVE/API2/ClusterConfig.pm |  19 +-
 data/PVE/CLI/pvecm.pm          |  17 +-
 data/PVE/Cluster.pm            | 724 +-------------------------------
 data/PVE/Cluster/Setup.pm      | 743 +++++++++++++++++++++++++++++++++
 5 files changed, 771 insertions(+), 734 deletions(-)
 create mode 100644 data/PVE/Cluster/Setup.pm

diff --git a/data/PVE/Cluster/Makefile b/data/PVE/Cluster/Makefile
index c0e9b85..0b25cfc 100644
--- a/data/PVE/Cluster/Makefile
+++ b/data/PVE/Cluster/Makefile
@@ -1,6 +1,6 @@
 PVEDIR=${DESTDIR}/usr/share/perl5/PVE
 
-SOURCES=IPCConst.pm
+SOURCES=IPCConst.pm Setup.pm
 
 .PHONY: install
 install: ${SOURCES}
diff --git a/data/PVE/API2/ClusterConfig.pm b/data/PVE/API2/ClusterConfig.pm
index 83053db..e6f47a6 100644
--- a/data/PVE/API2/ClusterConfig.pm
+++ b/data/PVE/API2/ClusterConfig.pm
@@ -11,6 +11,7 @@ use PVE::JSONSchema qw(get_standard_option);
 use PVE::Cluster;
 use PVE::APIClient::LWP;
 use PVE::Corosync;
+use PVE::Cluster::Setup;
 
 use IO::Socket::UNIX;
 
@@ -97,9 +98,9 @@ __PACKAGE__->register_method ({
 
 	my $code = sub {
 	    STDOUT->autoflush();
-	    PVE::Cluster::setup_sshd_config(1);
-	    PVE::Cluster::setup_rootsshconfig();
-	    PVE::Cluster::setup_ssh_keys();
+	    PVE::Cluster::Setup::setup_sshd_config(1);
+	    PVE::Cluster::Setup::setup_rootsshconfig();
+	    PVE::Cluster::Setup::setup_ssh_keys();
 
 	    PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile])
 		if !-f $authfile;
@@ -114,9 +115,9 @@ __PACKAGE__->register_method ({
 	    PVE::Corosync::atomic_write_conf($config);
 
 	    my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
-	    PVE::Cluster::ssh_merge_keys();
-	    PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
-	    PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+	    PVE::Cluster::Setup::ssh_merge_keys();
+	    PVE::Cluster::Setup::gen_pve_node_files($nodename, $local_ip_address);
+	    PVE::Cluster::Setup::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
 
 	    print "Restart corosync and cluster filesystem\n";
 	    PVE::Tools::run_command('systemctl restart corosync pve-cluster');
@@ -297,9 +298,9 @@ __PACKAGE__->register_method ({
 
 	    $param->{votes} = 1 if !defined($param->{votes});
 
-	    PVE::Cluster::gen_local_dirs($name);
+	    PVE::Cluster::Setup::gen_local_dirs($name);
 
-	    eval { PVE::Cluster::ssh_merge_keys(); };
+	    eval { PVE::Cluster::Setup::ssh_merge_keys(); };
 	    warn $@ if $@;
 
 	    $nodelist->{$name} = {
@@ -507,7 +508,7 @@ __PACKAGE__->register_method ({
 
 	my $worker = sub {
 	    STDOUT->autoflush();
-	    PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
+	    PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param);
 	    die $@ if $@;
 	};
 
diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
index 2c4c8f6..48c110b 100755
--- a/data/PVE/CLI/pvecm.pm
+++ b/data/PVE/CLI/pvecm.pm
@@ -14,6 +14,7 @@ use PVE::CLIHandler;
 use PVE::PTY;
 use PVE::API2::ClusterConfig;
 use PVE::Corosync;
+use PVE::Cluster::Setup;
 
 use base qw(PVE::CLIHandler);
 
@@ -359,7 +360,7 @@ __PACKAGE__->register_method ({
 	my $link0 = PVE::Cluster::parse_corosync_link($param->{link0});
 	my $link1 = PVE::Cluster::parse_corosync_link($param->{link1});
 
-	PVE::Cluster::assert_joinable($local_ip_address, $link0, $link1, $param->{force});
+	PVE::Cluster::Setup::assert_joinable($local_ip_address, $link0, $link1, $param->{force});
 
 	my $worker = sub {
 
@@ -371,7 +372,7 @@ __PACKAGE__->register_method ({
 		$param->{password} = $password;
 
 		my $local_cluster_lock = "/var/lock/pvecm.lock";
-		PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::join, $param);
+		PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param);
 
 		if (my $err = $@) {
 		    if (ref($err) eq 'PVE::APIClient::Exception' && defined($err->{code}) && $err->{code} == 501) {
@@ -385,12 +386,12 @@ __PACKAGE__->register_method ({
 
 	    # allow fallback to old ssh only join if wished or needed
 
-	    PVE::Cluster::setup_sshd_config();
-	    PVE::Cluster::setup_rootsshconfig();
-	    PVE::Cluster::setup_ssh_keys();
+	    PVE::Cluster::Setup::setup_sshd_config();
+	    PVE::Cluster::Setup::setup_rootsshconfig();
+	    PVE::Cluster::Setup::setup_ssh_keys();
 
 	    # make sure known_hosts is on local filesystem
-	    PVE::Cluster::ssh_unmerge_known_hosts();
+	    PVE::Cluster::Setup::ssh_unmerge_known_hosts();
 
 	    my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"];
 	    run_command($cmd, 'outfunc' => sub {}, 'errfunc' => sub {},
@@ -424,7 +425,7 @@ __PACKAGE__->register_method ({
 		my $corosync_conf = PVE::Tools::file_get_contents("$tmpdir/corosync.conf");
 		my $corosync_authkey = PVE::Tools::file_get_contents("$tmpdir/authkey");
 
-		PVE::Cluster::finish_join($host, $corosync_conf, $corosync_authkey);
+		PVE::Cluster::Setup::finish_join($host, $corosync_conf, $corosync_authkey);
 	    };
 	    my $err = $@;
 
@@ -565,7 +566,7 @@ __PACKAGE__->register_method ({
 	# IO (on /etc/pve) which can hang (uninterruptedly D state). That'd be
 	# no-good for ExecStartPost as it fails the whole service in this case
 	PVE::Tools::run_fork_with_timeout(30, sub {
-	    PVE::Cluster::updatecerts_and_ssh($param->@{qw(force silent)});
+	    PVE::Cluster::Setup::updatecerts_and_ssh($param->@{qw(force silent)});
 	});
 
 	return undef;
diff --git a/data/PVE/Cluster.pm b/data/PVE/Cluster.pm
index 6de7a27..28f59eb 100644
--- a/data/PVE/Cluster.pm
+++ b/data/PVE/Cluster.pm
@@ -3,18 +3,13 @@ package PVE::Cluster;
 use strict;
 use warnings;
 
-use Digest::HMAC_SHA1;
-use Digest::SHA;
 use Encode;
 use File::stat qw();
-use IO::File;
 use JSON;
-use MIME::Base64;
 use Net::SSLeay;
-use POSIX qw(EEXIST ENOENT);
+use POSIX qw(ENOENT);
 use Socket;
 use Storable qw(dclone);
-use UUID;
 
 use PVE::Certificate;
 use PVE::INotify;
@@ -43,31 +38,6 @@ my $lockdir = "/etc/pve/priv/lock";
 # cfs and corosync files
 my $dbfile = "/var/lib/pve-cluster/config.db";
 my $dbbackupdir = "/var/lib/pve-cluster/backup";
-my $localclusterdir = "/etc/corosync";
-my $localclusterconf = "$localclusterdir/corosync.conf";
-my $authfile = "$localclusterdir/authkey";
-my $clusterconf = "$basedir/corosync.conf";
-
-my $authprivkeyfn = "$authdir/authkey.key";
-my $authpubkeyfn = "$basedir/authkey.pub";
-my $pveca_key_fn = "$authdir/pve-root-ca.key";
-my $pveca_srl_fn = "$authdir/pve-root-ca.srl";
-my $pveca_cert_fn = "$basedir/pve-root-ca.pem";
-# this is just a secret accessable by the web browser
-# and is used for CSRF prevention
-my $pvewww_key_fn = "$basedir/pve-www.key";
-
-# ssh related files
-my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
-my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
-my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
-my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
-my $sshknownhosts = "/etc/pve/priv/known_hosts";
-my $sshauthkeys = "/etc/pve/priv/authorized_keys";
-my $sshd_config_fn = "/etc/ssh/sshd_config";
-my $rootsshauthkeys = "/root/.ssh/authorized_keys";
-my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
-my $rootsshconfig = "/root/.ssh/config";
 
 # this is just a readonly copy, the relevant one is in status.c from pmxcfs
 # observed files are the one we can get directly through IPCC, they are cached
@@ -97,19 +67,12 @@ my $observed = {
     'sdn.cfg.new' => 1,
 };
 
-# only write output if something fails
-sub run_silent_cmd {
-    my ($cmd) = @_;
+sub base_dir {
+    return $basedir;
+}
 
-    my $outbuf = '';
-    my $record = sub { $outbuf .= shift . "\n"; };
-
-    eval { run_command($cmd, outfunc => $record, errfunc => $record) };
-
-    if (my $err = $@) {
-	print STDERR $outbuf;
-	die $err;
-    }
+sub auth_dir {
+    return $authdir;
 }
 
 sub check_cfs_quorum {
@@ -136,255 +99,6 @@ sub check_cfs_is_mounted {
     return $res;
 }
 
-sub gen_local_dirs {
-    my ($nodename) = @_;
-
-    check_cfs_is_mounted();
-
-    my @required_dirs = (
-	"$basedir/priv",
-	"$basedir/nodes",
-	"$basedir/nodes/$nodename",
-	"$basedir/nodes/$nodename/lxc",
-	"$basedir/nodes/$nodename/qemu-server",
-	"$basedir/nodes/$nodename/openvz",
-	"$basedir/nodes/$nodename/priv");
-
-    foreach my $dir (@required_dirs) {
-	if (! -d $dir) {
-	    mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
-	}
-    }
-}
-
-sub gen_auth_key {
-
-    return if -f "$authprivkeyfn";
-
-    check_cfs_is_mounted();
-
-    cfs_lock_authkey(undef, sub {
-	mkdir $authdir || $! == EEXIST || die "unable to create dir '$authdir' - $!\n";
-
-	run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
-
-	run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
-    });
-
-    die "$@\n" if $@;
-}
-
-sub gen_pveca_key {
-
-    return if -f $pveca_key_fn;
-
-    eval {
-	run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
-    };
-
-    die "unable to generate pve ca key:\n$@" if $@;
-}
-
-sub gen_pveca_cert {
-
-    if (-f $pveca_key_fn && -f $pveca_cert_fn) {
-	return 0;
-    }
-
-    gen_pveca_key();
-
-    # we try to generate an unique 'subject' to avoid browser problems
-    # (reused serial numbers, ..)
-    my $uuid;
-    UUID::generate($uuid);
-    my $uuid_str;
-    UUID::unparse($uuid, $uuid_str);
-
-    eval {
-	# wrap openssl with faketime to prevent bug #904
-	run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
-			'-days', '3650', '-new', '-x509', '-nodes', '-key',
-			$pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
-			"/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
-    };
-
-    die "generating pve root certificate failed:\n$@" if $@;
-
-    return 1;
-}
-
-sub gen_pve_ssl_key {
-    my ($nodename) = @_;
-
-    die "no node name specified" if !$nodename;
-
-    my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
-
-    return if -f $pvessl_key_fn;
-
-    eval {
-	run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
-    };
-
-    die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
-}
-
-sub gen_pve_www_key {
-
-    return if -f $pvewww_key_fn;
-
-    eval {
-	run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
-    };
-
-    die "unable to generate pve www key:\n$@" if $@;
-}
-
-sub update_serial {
-    my ($serial) = @_;
-
-    PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
-}
-
-sub gen_pve_ssl_cert {
-    my ($force, $nodename, $ip) = @_;
-
-    die "no node name specified" if !$nodename;
-    die "no IP specified" if !$ip;
-
-    my $pvessl_cert_fn = "$basedir/nodes/$nodename/pve-ssl.pem";
-
-    return if !$force && -f $pvessl_cert_fn;
-
-    my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
-
-    my $rc = PVE::INotify::read_file('resolvconf');
-
-    $names .= ",IP:$ip";
-
-    my $fqdn = $nodename;
-
-    $names .= ",DNS:$nodename";
-
-    if ($rc && $rc->{search}) {
-	$fqdn = $nodename . "." . $rc->{search};
-	$names .= ",DNS:$fqdn";
-    }
-
-    my $sslconf = <<__EOD;
-RANDFILE = /root/.rnd
-extensions = v3_req
-
-[ req ]
-default_bits = 2048
-distinguished_name = req_distinguished_name
-req_extensions = v3_req
-prompt = no
-string_mask = nombstr
-
-[ req_distinguished_name ]
-organizationalUnitName = PVE Cluster Node
-organizationName = Proxmox Virtual Environment
-commonName = $fqdn
-
-[ v3_req ]
-basicConstraints = CA:FALSE
-extendedKeyUsage = serverAuth
-subjectAltName = $names
-__EOD
-
-    my $cfgfn = "/tmp/pvesslconf-$$.tmp";
-    my $fh = IO::File->new ($cfgfn, "w");
-    print $fh $sslconf;
-    close ($fh);
-
-    my $reqfn = "/tmp/pvecertreq-$$.tmp";
-    unlink $reqfn;
-
-    my $pvessl_key_fn = "$basedir/nodes/$nodename/pve-ssl.key";
-    eval {
-	run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
-			'-key', $pvessl_key_fn, '-out', $reqfn]);
-    };
-
-    if (my $err = $@) {
-	unlink $reqfn;
-	unlink $cfgfn;
-	die "unable to generate pve certificate request:\n$err";
-    }
-
-    update_serial("0000000000000000") if ! -f $pveca_srl_fn;
-
-    eval {
-	# wrap openssl with faketime to prevent bug #904
-	run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
-			'-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
-			'-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
-			'-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
-    };
-
-    if (my $err = $@) {
-	unlink $reqfn;
-	unlink $cfgfn;
-	die "unable to generate pve ssl certificate:\n$err";
-    }
-
-    unlink $cfgfn;
-    unlink $reqfn;
-}
-
-sub gen_pve_node_files {
-    my ($nodename, $ip, $opt_force) = @_;
-
-    gen_local_dirs($nodename);
-
-    gen_auth_key();
-
-    # make sure we have a (cluster wide) secret
-    # for CSRFR prevention
-    gen_pve_www_key();
-
-    # make sure we have a (per node) private key
-    gen_pve_ssl_key($nodename);
-
-    # make sure we have a CA
-    my $force = gen_pveca_cert();
-
-    $force = 1 if $opt_force;
-
-    gen_pve_ssl_cert($force, $nodename, $ip);
-}
-
-my $vzdump_cron_dummy = <<__EOD;
-# cluster wide vzdump cron schedule
-# Atomatically generated file - do not edit
-
-PATH="/usr/sbin:/usr/bin:/sbin:/bin"
-
-__EOD
-
-sub gen_pve_vzdump_symlink {
-
-    my $filename = "/etc/pve/vzdump.cron";
-
-    my $link_fn = "/etc/cron.d/vzdump";
-
-    if ((-f $filename) && (! -l $link_fn)) {
-	rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
-	symlink($filename, $link_fn);
-    }
-}
-
-sub gen_pve_vzdump_files {
-
-    my $filename = "/etc/pve/vzdump.cron";
-
-    PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
-	if ! -f $filename;
-
-    gen_pve_vzdump_symlink();
-};
-
 my $versions = {};
 my $vmlist = {};
 my $clinfo = {};
@@ -1038,247 +752,6 @@ sub get_local_migration_ip {
     return undef;
 };
 
-# ssh related utility functions
-
-sub ssh_merge_keys {
-    # remove duplicate keys in $sshauthkeys
-    # ssh-copy-id simply add keys, so the file can grow to large
-
-    my $data = '';
-    if (-f $sshauthkeys) {
-	$data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
-	chomp($data);
-    }
-
-    my $found_backup;
-    if (-f $rootsshauthkeysbackup) {
-	$data .= "\n";
-	$data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
-	chomp($data);
-	$found_backup = 1;
-    }
-
-    # always add ourself
-    if (-f $ssh_rsa_id) {
-	my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
-	chomp($pub);
-	$data .= "\n$pub\n";
-    }
-
-    my $newdata = "";
-    my $vhash = {};
-    my @lines = split(/\n/, $data);
-    foreach my $line (@lines) {
-	if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
-            next if $vhash->{$3}++;
-	}
-	$newdata .= "$line\n";
-    }
-
-    PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
-
-    if ($found_backup && -l $rootsshauthkeys) {
-	# everything went well, so we can remove the backup
-	unlink $rootsshauthkeysbackup;
-    }
-}
-
-sub setup_sshd_config {
-    my () = @_;
-
-    my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
-
-    return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
-
-    if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
-	chomp $conf;
-	$conf .= "\nPermitRootLogin yes\n";
-    }
-
-    PVE::Tools::file_set_contents($sshd_config_fn, $conf);
-
-    PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
-}
-
-sub setup_rootsshconfig {
-
-    # create ssh key if it does not exist
-    if (! -f $ssh_rsa_id) {
-	mkdir '/root/.ssh/';
-	system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
-    }
-
-    # create ssh config if it does not exist
-    if (! -f $rootsshconfig) {
-        mkdir '/root/.ssh';
-        if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
-            # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
-	    # changed order to put AES before Chacha20 (most hardware has AESNI)
-            print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
-            close($fh);
-        }
-    }
-}
-
-sub setup_ssh_keys {
-
-    mkdir $authdir;
-
-    my $import_ok;
-
-    if (! -f $sshauthkeys) {
-	my $old;
-	if (-f $rootsshauthkeys) {
-	    $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
-	}
-	if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
-	    PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
-	    close($fh);
-	    $import_ok = 1;
-	}
-    }
-
-    warn "can't create shared ssh key database '$sshauthkeys'\n"
-	if ! -f $sshauthkeys;
-
-    if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
-	if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
-	    warn "rename $rootsshauthkeys failed - $!\n";
-	}
-    }
-
-    if (! -l $rootsshauthkeys) {
-	symlink $sshauthkeys, $rootsshauthkeys;
-    }
-
-    if (! -l $rootsshauthkeys) {
-	warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
-    } else {
-	unlink $rootsshauthkeysbackup if $import_ok;
-    }
-}
-
-sub ssh_unmerge_known_hosts {
-    return if ! -l $sshglobalknownhosts;
-
-    my $old = '';
-    $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
-	if -f $sshknownhosts;
-
-    PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
-}
-
-sub ssh_merge_known_hosts {
-    my ($nodename, $ip_address, $createLink) = @_;
-
-    die "no node name specified" if !$nodename;
-    die "no ip address specified" if !$ip_address;
-
-    # ssh lowercases hostnames (aliases) before comparision, so we need too
-    $nodename = lc($nodename);
-    $ip_address = lc($ip_address);
-
-    mkdir $authdir;
-
-    if (! -f $sshknownhosts) {
-	if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
-	    close($fh);
-	}
-    }
-
-    my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
-
-    my $new = '';
-
-    if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
-	$new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
-    }
-
-    my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
-    # Note: file sometimes containe emty lines at start, so we use multiline match
-    die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
-    $hostkey = $1;
-
-    my $data = '';
-    my $vhash = {};
-
-    my $found_nodename;
-    my $found_local_ip;
-
-    my $merge_line = sub {
-	my ($line, $all) = @_;
-
-	return if $line =~ m/^\s*$/; # skip empty lines
-	return if $line =~ m/^#/; # skip comments
-
-	if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
-	    my $key = $1;
-	    my $rsakey = $2;
-	    if (!$vhash->{$key}) {
-		$vhash->{$key} = 1;
-		if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
-		    my $salt = decode_base64($1);
-		    my $digest = $2;
-		    my $hmac = Digest::HMAC_SHA1->new($salt);
-		    $hmac->add($nodename);
-		    my $hd = $hmac->b64digest . '=';
-		    if ($digest eq $hd) {
-			if ($rsakey eq $hostkey) {
-			    $found_nodename = 1;
-			    $data .= $line;
-			}
-			return;
-		    }
-		    $hmac = Digest::HMAC_SHA1->new($salt);
-		    $hmac->add($ip_address);
-		    $hd = $hmac->b64digest . '=';
-		    if ($digest eq $hd) {
-			if ($rsakey eq $hostkey) {
-			    $found_local_ip = 1;
-			    $data .= $line;
-			}
-			return;
-		    }
-		} else {
-		    $key = lc($key); # avoid duplicate entries, ssh compares lowercased
-		    if ($key eq $ip_address) {
-			$found_local_ip = 1 if $rsakey eq $hostkey;
-		    } elsif ($key eq $nodename) {
-			$found_nodename = 1 if $rsakey eq $hostkey;
-		    }
-		}
-		$data .= $line;
-	    }
-	} elsif ($all) {
-	    $data .= $line;
-	}
-    };
-
-    while ($old && $old =~ s/^((.*?)(\n|$))//) {
-	my $line = "$2\n";
-	&$merge_line($line, 1);
-    }
-
-    while ($new && $new =~ s/^((.*?)(\n|$))//) {
-	my $line = "$2\n";
-	&$merge_line($line);
-    }
-
-    # add our own key if not already there
-    $data .= "$nodename $hostkey\n" if !$found_nodename;
-    $data .= "$ip_address $hostkey\n" if !$found_local_ip;
-
-    PVE::Tools::file_set_contents($sshknownhosts, $data);
-
-    return if !$createLink;
-
-    unlink $sshglobalknownhosts;
-    symlink $sshknownhosts, $sshglobalknownhosts;
-
-    warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
-	if ! -l $sshglobalknownhosts;
-
-}
 
 my $migration_format = {
     type => {
@@ -1742,63 +1215,8 @@ sub parse_corosync_link {
     return PVE::JSONSchema::parse_property_string($corosync_link_format, $value);
 }
 
-sub assert_joinable {
-    my ($local_addr, $link0, $link1, $force) = @_;
-
-    my $errors = '';
-    my $error = sub { $errors .= "* $_[0]\n"; };
-
-    if (-f $authfile) {
-	$error->("authentication key '$authfile' already exists");
-    }
-
-    if (-f $clusterconf)  {
-	$error->("cluster config '$clusterconf' already exists");
-    }
-
-    my $vmlist = get_vmlist();
-    if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
-	$error->("this host already contains virtual guests");
-    }
-
-    if (run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
-	$error->("corosync is already running, is this node already in a cluster?!");
-    }
-
-    # check if corosync ring IPs are configured on the current nodes interfaces
-    my $check_ip = sub {
-	my $ip = shift // return;
-	my $logid = shift;
-	if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
-	    my $host = $ip;
-	    eval { $ip = PVE::Network::get_ip_from_hostname($host); };
-	    if ($@) {
-		$error->("$logid: cannot use '$host': $@\n") ;
-		return;
-	    }
-	}
-
-	my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
-	my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
-
-	$error->("$logid: cannot use IP '$ip', it must be configured exactly once on local node!\n")
-	    if (scalar(@$configured_ips) != 1);
-    };
-
-    $check_ip->($local_addr, 'local node address');
-    $check_ip->($link0->{address}, 'ring0') if defined($link0);
-    $check_ip->($link1->{address}, 'ring1') if defined($link1);
-
-    if ($errors) {
-	warn "detected the following error(s):\n$errors";
-	die "Check if node may join a cluster failed!\n" if !$force;
-    }
-}
-
 # NOTE: filesystem must be offline here, no DB changes allowed
-my $backup_cfs_database = sub {
-    my ($dbfile) = @_;
-
+sub cfs_backup_database {
     mkdir $dbbackupdir;
 
     my $ctime = time();
@@ -1819,134 +1237,8 @@ my $backup_cfs_database = sub {
 	    unlink $1;
 	}
     }
-};
 
-sub join {
-    my ($param) = @_;
-
-    my $nodename = PVE::INotify::nodename();
-    my $local_ip_address = remote_node_ip($nodename);
-
-    my $link0 = parse_corosync_link($param->{link0});
-    my $link1 = parse_corosync_link($param->{link1});
-
-    # check if we can join with the given parameters and current node state
-    assert_joinable($local_ip_address, $link0, $link1, $param->{force});
-
-    setup_sshd_config();
-    setup_rootsshconfig();
-    setup_ssh_keys();
-
-    # make sure known_hosts is on local filesystem
-    ssh_unmerge_known_hosts();
-
-    my $host = $param->{hostname};
-    my $conn_args = {
-	username => 'root at pam',
-	password => $param->{password},
-	cookie_name => 'PVEAuthCookie',
-	protocol => 'https',
-	host => $host,
-	port => 8006,
-    };
-
-    if (my $fp = $param->{fingerprint}) {
-	$conn_args->{cached_fingerprints} = { uc($fp) => 1 };
-    } else {
-	# API schema ensures that we can only get here from CLI handler
-	$conn_args->{manual_verification} = 1;
-    }
-
-    print "Establishing API connection with host '$host'\n";
-
-    my $conn = PVE::APIClient::LWP->new(%$conn_args);
-    $conn->login();
-
-    # login raises an exception on failure, so if we get here we're good
-    print "Login succeeded.\n";
-
-    my $args = {};
-    $args->{force} = $param->{force} if defined($param->{force});
-    $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
-    $args->{votes} = $param->{votes} if defined($param->{votes});
-    # just pass the un-parsed string through, or as we've address as the
-    # default_key, we can just pass the fallback directly too
-    $args->{link0} = $param->{link0} // $local_ip_address;
-    $args->{link1} = $param->{link1} if defined($param->{link1});
-
-    print "Request addition of this node\n";
-    my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
-
-    print "Join request OK, finishing setup locally\n";
-
-    # added successfuly - now prepare local node
-    finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
-}
-
-sub finish_join {
-    my ($nodename, $corosync_conf, $corosync_authkey) = @_;
-
-    mkdir "$localclusterdir";
-    PVE::Tools::file_set_contents($authfile, $corosync_authkey);
-    PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
-
-    print "stopping pve-cluster service\n";
-    my $cmd = ['systemctl', 'stop', 'pve-cluster'];
-    run_command($cmd, errmsg => "can't stop pve-cluster service");
-
-    $backup_cfs_database->($dbfile);
-    unlink $dbfile;
-
-    $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
-    run_command($cmd, errmsg => "starting pve-cluster failed");
-
-    # wait for quorum
-    my $printqmsg = 1;
-    while (!check_cfs_quorum(1)) {
-	if ($printqmsg) {
-	    print "waiting for quorum...";
-	    STDOUT->flush();
-	    $printqmsg = 0;
-	}
-	sleep(1);
-    }
-    print "OK\n" if !$printqmsg;
-
-    updatecerts_and_ssh(1);
-
-    print "generated new node certificate, restart pveproxy and pvedaemon services\n";
-    run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
-
-    print "successfully added node '$nodename' to cluster.\n";
-}
-
-sub updatecerts_and_ssh {
-    my ($force_new_cert, $silent) = @_;
-
-    my $p = sub { print "$_[0]\n" if !$silent };
-
-    setup_rootsshconfig();
-
-    gen_pve_vzdump_symlink();
-
-    if (!check_cfs_quorum(1)) {
-	return undef if $silent;
-	die "no quorum - unable to update files\n";
-    }
-
-    setup_ssh_keys();
-
-    my $nodename = PVE::INotify::nodename();
-    my $local_ip_address = remote_node_ip($nodename);
-
-    $p->("(re)generate node files");
-    $p->("generate new node certificate") if $force_new_cert;
-    gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
-
-    $p->("merge authorized SSH keys and known hosts");
-    ssh_merge_keys();
-    ssh_merge_known_hosts($nodename, $local_ip_address, 1);
-    gen_pve_vzdump_files();
+    return $dbfile;
 }
 
 1;
diff --git a/data/PVE/Cluster/Setup.pm b/data/PVE/Cluster/Setup.pm
new file mode 100644
index 0000000..e81a110
--- /dev/null
+++ b/data/PVE/Cluster/Setup.pm
@@ -0,0 +1,743 @@
+package PVE::Cluster::Setup;
+
+use strict;
+use warnings;
+
+use Digest::HMAC_SHA1;
+use Digest::SHA;
+use IO::File;
+use MIME::Base64;
+use Net::IP;
+use UUID;
+use POSIX qw(EEXIST);
+
+use PVE::APIClient::LWP;
+use PVE::Cluster;
+use PVE::INotify;
+use PVE::JSONSchema;
+use PVE::Network;
+use PVE::Tools;
+
+my $pmxcfs_base_dir = PVE::Cluster::base_dir();
+my $pmxcfs_auth_dir = PVE::Cluster::auth_dir();
+
+# only write output if something fails
+sub run_silent_cmd {
+    my ($cmd) = @_;
+
+    my $outbuf = '';
+    my $record = sub { $outbuf .= shift . "\n"; };
+
+    eval { PVE::Tools::run_command($cmd, outfunc => $record, errfunc => $record) };
+
+    if (my $err = $@) {
+	print STDERR $outbuf;
+	die $err;
+    }
+}
+
+# Corosync related files
+my $localclusterdir = "/etc/corosync";
+my $localclusterconf = "$localclusterdir/corosync.conf";
+my $authfile = "$localclusterdir/authkey";
+my $clusterconf = "$pmxcfs_base_dir/corosync.conf";
+
+# CA/certificate related files
+my $pveca_key_fn = "$pmxcfs_auth_dir/pve-root-ca.key";
+my $pveca_srl_fn = "$pmxcfs_auth_dir/pve-root-ca.srl";
+my $pveca_cert_fn = "$pmxcfs_base_dir/pve-root-ca.pem";
+# this is just a secret accessable by the web browser
+# and is used for CSRF prevention
+my $pvewww_key_fn = "$pmxcfs_base_dir/pve-www.key";
+
+# ssh related files
+my $ssh_rsa_id_priv = "/root/.ssh/id_rsa";
+my $ssh_rsa_id = "/root/.ssh/id_rsa.pub";
+my $ssh_host_rsa_id = "/etc/ssh/ssh_host_rsa_key.pub";
+my $sshglobalknownhosts = "/etc/ssh/ssh_known_hosts";
+my $sshknownhosts = "$pmxcfs_auth_dir/known_hosts";
+my $sshauthkeys = "$pmxcfs_auth_dir/authorized_keys";
+my $sshd_config_fn = "/etc/ssh/sshd_config";
+my $rootsshauthkeys = "/root/.ssh/authorized_keys";
+my $rootsshauthkeysbackup = "${rootsshauthkeys}.org";
+my $rootsshconfig = "/root/.ssh/config";
+
+# ssh related utility functions
+
+sub ssh_merge_keys {
+    # remove duplicate keys in $sshauthkeys
+    # ssh-copy-id simply add keys, so the file can grow to large
+
+    my $data = '';
+    if (-f $sshauthkeys) {
+	$data = PVE::Tools::file_get_contents($sshauthkeys, 128*1024);
+	chomp($data);
+    }
+
+    my $found_backup;
+    if (-f $rootsshauthkeysbackup) {
+	$data .= "\n";
+	$data .= PVE::Tools::file_get_contents($rootsshauthkeysbackup, 128*1024);
+	chomp($data);
+	$found_backup = 1;
+    }
+
+    # always add ourself
+    if (-f $ssh_rsa_id) {
+	my $pub = PVE::Tools::file_get_contents($ssh_rsa_id);
+	chomp($pub);
+	$data .= "\n$pub\n";
+    }
+
+    my $newdata = "";
+    my $vhash = {};
+    my @lines = split(/\n/, $data);
+    foreach my $line (@lines) {
+	if ($line !~ /^#/ && $line =~ m/(^|\s)ssh-(rsa|dsa)\s+(\S+)\s+\S+$/) {
+            next if $vhash->{$3}++;
+	}
+	$newdata .= "$line\n";
+    }
+
+    PVE::Tools::file_set_contents($sshauthkeys, $newdata, 0600);
+
+    if ($found_backup && -l $rootsshauthkeys) {
+	# everything went well, so we can remove the backup
+	unlink $rootsshauthkeysbackup;
+    }
+}
+
+sub setup_sshd_config {
+    my () = @_;
+
+    my $conf = PVE::Tools::file_get_contents($sshd_config_fn);
+
+    return if $conf =~ m/^PermitRootLogin\s+yes\s*$/m;
+
+    if ($conf !~ s/^#?PermitRootLogin.*$/PermitRootLogin yes/m) {
+	chomp $conf;
+	$conf .= "\nPermitRootLogin yes\n";
+    }
+
+    PVE::Tools::file_set_contents($sshd_config_fn, $conf);
+
+    PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'sshd']);
+}
+
+sub setup_rootsshconfig {
+
+    # create ssh key if it does not exist
+    if (! -f $ssh_rsa_id) {
+	mkdir '/root/.ssh/';
+	system ("echo|ssh-keygen -t rsa -N '' -b 2048 -f ${ssh_rsa_id_priv}");
+    }
+
+    # create ssh config if it does not exist
+    if (! -f $rootsshconfig) {
+        mkdir '/root/.ssh';
+        if (my $fh = IO::File->new($rootsshconfig, O_CREAT|O_WRONLY|O_EXCL, 0640)) {
+            # this is the default ciphers list from Debian's OpenSSH package (OpenSSH_7.4p1 Debian-10, OpenSSL 1.0.2k  26 Jan 2017)
+	    # changed order to put AES before Chacha20 (most hardware has AESNI)
+            print $fh "Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm\@openssh.com,aes256-gcm\@openssh.com,chacha20-poly1305\@openssh.com\n";
+            close($fh);
+        }
+    }
+}
+
+sub setup_ssh_keys {
+
+    mkdir $pmxcfs_auth_dir;
+
+    my $import_ok;
+
+    if (! -f $sshauthkeys) {
+	my $old;
+	if (-f $rootsshauthkeys) {
+	    $old = PVE::Tools::file_get_contents($rootsshauthkeys, 128*1024);
+	}
+	if (my $fh = IO::File->new ($sshauthkeys, O_CREAT|O_WRONLY|O_EXCL, 0400)) {
+	    PVE::Tools::safe_print($sshauthkeys, $fh, $old) if $old;
+	    close($fh);
+	    $import_ok = 1;
+	}
+    }
+
+    warn "can't create shared ssh key database '$sshauthkeys'\n"
+	if ! -f $sshauthkeys;
+
+    if (-f $rootsshauthkeys && ! -l $rootsshauthkeys) {
+	if (!rename($rootsshauthkeys , $rootsshauthkeysbackup)) {
+	    warn "rename $rootsshauthkeys failed - $!\n";
+	}
+    }
+
+    if (! -l $rootsshauthkeys) {
+	symlink $sshauthkeys, $rootsshauthkeys;
+    }
+
+    if (! -l $rootsshauthkeys) {
+	warn "can't create symlink for ssh keys '$rootsshauthkeys' -> '$sshauthkeys'\n";
+    } else {
+	unlink $rootsshauthkeysbackup if $import_ok;
+    }
+}
+
+sub ssh_unmerge_known_hosts {
+    return if ! -l $sshglobalknownhosts;
+
+    my $old = '';
+    $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024)
+	if -f $sshknownhosts;
+
+    PVE::Tools::file_set_contents($sshglobalknownhosts, $old);
+}
+
+sub ssh_merge_known_hosts {
+    my ($nodename, $ip_address, $createLink) = @_;
+
+    die "no node name specified" if !$nodename;
+    die "no ip address specified" if !$ip_address;
+
+    # ssh lowercases hostnames (aliases) before comparision, so we need too
+    $nodename = lc($nodename);
+    $ip_address = lc($ip_address);
+
+    mkdir $pmxcfs_auth_dir;
+
+    if (! -f $sshknownhosts) {
+	if (my $fh = IO::File->new($sshknownhosts, O_CREAT|O_WRONLY|O_EXCL, 0600)) {
+	    close($fh);
+	}
+    }
+
+    my $old = PVE::Tools::file_get_contents($sshknownhosts, 128*1024);
+
+    my $new = '';
+
+    if ((! -l $sshglobalknownhosts) && (-f $sshglobalknownhosts)) {
+	$new = PVE::Tools::file_get_contents($sshglobalknownhosts, 128*1024);
+    }
+
+    my $hostkey = PVE::Tools::file_get_contents($ssh_host_rsa_id);
+    # Note: file sometimes containe emty lines at start, so we use multiline match
+    die "can't parse $ssh_host_rsa_id" if $hostkey !~ m/^(ssh-rsa\s\S+)(\s.*)?$/m;
+    $hostkey = $1;
+
+    my $data = '';
+    my $vhash = {};
+
+    my $found_nodename;
+    my $found_local_ip;
+
+    my $merge_line = sub {
+	my ($line, $all) = @_;
+
+	return if $line =~ m/^\s*$/; # skip empty lines
+	return if $line =~ m/^#/; # skip comments
+
+	if ($line =~ m/^(\S+)\s(ssh-rsa\s\S+)(\s.*)?$/) {
+	    my $key = $1;
+	    my $rsakey = $2;
+	    if (!$vhash->{$key}) {
+		$vhash->{$key} = 1;
+		if ($key =~ m/\|1\|([^\|\s]+)\|([^\|\s]+)$/) {
+		    my $salt = decode_base64($1);
+		    my $digest = $2;
+		    my $hmac = Digest::HMAC_SHA1->new($salt);
+		    $hmac->add($nodename);
+		    my $hd = $hmac->b64digest . '=';
+		    if ($digest eq $hd) {
+			if ($rsakey eq $hostkey) {
+			    $found_nodename = 1;
+			    $data .= $line;
+			}
+			return;
+		    }
+		    $hmac = Digest::HMAC_SHA1->new($salt);
+		    $hmac->add($ip_address);
+		    $hd = $hmac->b64digest . '=';
+		    if ($digest eq $hd) {
+			if ($rsakey eq $hostkey) {
+			    $found_local_ip = 1;
+			    $data .= $line;
+			}
+			return;
+		    }
+		} else {
+		    $key = lc($key); # avoid duplicate entries, ssh compares lowercased
+		    if ($key eq $ip_address) {
+			$found_local_ip = 1 if $rsakey eq $hostkey;
+		    } elsif ($key eq $nodename) {
+			$found_nodename = 1 if $rsakey eq $hostkey;
+		    }
+		}
+		$data .= $line;
+	    }
+	} elsif ($all) {
+	    $data .= $line;
+	}
+    };
+
+    while ($old && $old =~ s/^((.*?)(\n|$))//) {
+	my $line = "$2\n";
+	&$merge_line($line, 1);
+    }
+
+    while ($new && $new =~ s/^((.*?)(\n|$))//) {
+	my $line = "$2\n";
+	&$merge_line($line);
+    }
+
+    # add our own key if not already there
+    $data .= "$nodename $hostkey\n" if !$found_nodename;
+    $data .= "$ip_address $hostkey\n" if !$found_local_ip;
+
+    PVE::Tools::file_set_contents($sshknownhosts, $data);
+
+    return if !$createLink;
+
+    unlink $sshglobalknownhosts;
+    symlink $sshknownhosts, $sshglobalknownhosts;
+
+    warn "can't create symlink for ssh known hosts '$sshglobalknownhosts' -> '$sshknownhosts'\n"
+	if ! -l $sshglobalknownhosts;
+
+}
+
+# directory and file creation
+
+sub gen_local_dirs {
+    my ($nodename) = @_;
+
+    PVE::Cluster::check_cfs_is_mounted();
+
+    my @required_dirs = (
+	"$pmxcfs_base_dir/priv",
+	"$pmxcfs_base_dir/nodes",
+	"$pmxcfs_base_dir/nodes/$nodename",
+	"$pmxcfs_base_dir/nodes/$nodename/lxc",
+	"$pmxcfs_base_dir/nodes/$nodename/qemu-server",
+	"$pmxcfs_base_dir/nodes/$nodename/openvz",
+	"$pmxcfs_base_dir/nodes/$nodename/priv");
+
+    foreach my $dir (@required_dirs) {
+	if (! -d $dir) {
+	    mkdir($dir) || $! == EEXIST || die "unable to create directory '$dir' - $!\n";
+	}
+    }
+}
+
+sub gen_auth_key {
+    my $authprivkeyfn = "$pmxcfs_auth_dir/authkey.key";
+    my $authpubkeyfn = "$pmxcfs_base_dir/authkey.pub";
+
+    return if -f "$authprivkeyfn";
+
+    PVE::Cluster::check_cfs_is_mounted();
+
+    PVE::Cluster::cfs_lock_authkey(undef, sub {
+	mkdir $pmxcfs_auth_dir || $! == EEXIST || die "unable to create dir '$pmxcfs_auth_dir' - $!\n";
+
+	run_silent_cmd(['openssl', 'genrsa', '-out', $authprivkeyfn, '2048']);
+
+	run_silent_cmd(['openssl', 'rsa', '-in', $authprivkeyfn, '-pubout', '-out', $authpubkeyfn]);
+    });
+
+    die "$@\n" if $@;
+}
+
+sub gen_pveca_key {
+
+    return if -f $pveca_key_fn;
+
+    eval {
+	run_silent_cmd(['openssl', 'genrsa', '-out', $pveca_key_fn, '4096']);
+    };
+
+    die "unable to generate pve ca key:\n$@" if $@;
+}
+
+sub gen_pveca_cert {
+
+    if (-f $pveca_key_fn && -f $pveca_cert_fn) {
+	return 0;
+    }
+
+    gen_pveca_key();
+
+    # we try to generate an unique 'subject' to avoid browser problems
+    # (reused serial numbers, ..)
+    my $uuid;
+    UUID::generate($uuid);
+    my $uuid_str;
+    UUID::unparse($uuid, $uuid_str);
+
+    eval {
+	# wrap openssl with faketime to prevent bug #904
+	run_silent_cmd(['faketime', 'yesterday', 'openssl', 'req', '-batch',
+			'-days', '3650', '-new', '-x509', '-nodes', '-key',
+			$pveca_key_fn, '-out', $pveca_cert_fn, '-subj',
+			"/CN=Proxmox Virtual Environment/OU=$uuid_str/O=PVE Cluster Manager CA/"]);
+    };
+
+    die "generating pve root certificate failed:\n$@" if $@;
+
+    return 1;
+}
+
+sub gen_pve_ssl_key {
+    my ($nodename) = @_;
+
+    die "no node name specified" if !$nodename;
+
+    my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
+
+    return if -f $pvessl_key_fn;
+
+    eval {
+	run_silent_cmd(['openssl', 'genrsa', '-out', $pvessl_key_fn, '2048']);
+    };
+
+    die "unable to generate pve ssl key for node '$nodename':\n$@" if $@;
+}
+
+sub gen_pve_www_key {
+
+    return if -f $pvewww_key_fn;
+
+    eval {
+	run_silent_cmd(['openssl', 'genrsa', '-out', $pvewww_key_fn, '2048']);
+    };
+
+    die "unable to generate pve www key:\n$@" if $@;
+}
+
+sub update_serial {
+    my ($serial) = @_;
+
+    PVE::Tools::file_set_contents($pveca_srl_fn, $serial);
+}
+
+sub gen_pve_ssl_cert {
+    my ($force, $nodename, $ip) = @_;
+
+    die "no node name specified" if !$nodename;
+    die "no IP specified" if !$ip;
+
+    my $pvessl_cert_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.pem";
+
+    return if !$force && -f $pvessl_cert_fn;
+
+    my $names = "IP:127.0.0.1,IP:::1,DNS:localhost";
+
+    my $rc = PVE::INotify::read_file('resolvconf');
+
+    $names .= ",IP:$ip";
+
+    my $fqdn = $nodename;
+
+    $names .= ",DNS:$nodename";
+
+    if ($rc && $rc->{search}) {
+	$fqdn = $nodename . "." . $rc->{search};
+	$names .= ",DNS:$fqdn";
+    }
+
+    my $sslconf = <<__EOD;
+RANDFILE = /root/.rnd
+extensions = v3_req
+
+[ req ]
+default_bits = 2048
+distinguished_name = req_distinguished_name
+req_extensions = v3_req
+prompt = no
+string_mask = nombstr
+
+[ req_distinguished_name ]
+organizationalUnitName = PVE Cluster Node
+organizationName = Proxmox Virtual Environment
+commonName = $fqdn
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+extendedKeyUsage = serverAuth
+subjectAltName = $names
+__EOD
+
+    my $cfgfn = "/tmp/pvesslconf-$$.tmp";
+    my $fh = IO::File->new ($cfgfn, "w");
+    print $fh $sslconf;
+    close ($fh);
+
+    my $reqfn = "/tmp/pvecertreq-$$.tmp";
+    unlink $reqfn;
+
+    my $pvessl_key_fn = "$pmxcfs_base_dir/nodes/$nodename/pve-ssl.key";
+    eval {
+	run_silent_cmd(['openssl', 'req', '-batch', '-new', '-config', $cfgfn,
+			'-key', $pvessl_key_fn, '-out', $reqfn]);
+    };
+
+    if (my $err = $@) {
+	unlink $reqfn;
+	unlink $cfgfn;
+	die "unable to generate pve certificate request:\n$err";
+    }
+
+    update_serial("0000000000000000") if ! -f $pveca_srl_fn;
+
+    eval {
+	# wrap openssl with faketime to prevent bug #904
+	run_silent_cmd(['faketime', 'yesterday', 'openssl', 'x509', '-req',
+			'-in', $reqfn, '-days', '3650', '-out', $pvessl_cert_fn,
+			'-CAkey', $pveca_key_fn, '-CA', $pveca_cert_fn,
+			'-CAserial', $pveca_srl_fn, '-extfile', $cfgfn]);
+    };
+
+    if (my $err = $@) {
+	unlink $reqfn;
+	unlink $cfgfn;
+	die "unable to generate pve ssl certificate:\n$err";
+    }
+
+    unlink $cfgfn;
+    unlink $reqfn;
+}
+
+sub gen_pve_node_files {
+    my ($nodename, $ip, $opt_force) = @_;
+
+    gen_local_dirs($nodename);
+
+    gen_auth_key();
+
+    # make sure we have a (cluster wide) secret
+    # for CSRFR prevention
+    gen_pve_www_key();
+
+    # make sure we have a (per node) private key
+    gen_pve_ssl_key($nodename);
+
+    # make sure we have a CA
+    my $force = gen_pveca_cert();
+
+    $force = 1 if $opt_force;
+
+    gen_pve_ssl_cert($force, $nodename, $ip);
+}
+
+my $vzdump_cron_dummy = <<__EOD;
+# cluster wide vzdump cron schedule
+# Atomatically generated file - do not edit
+
+PATH="/usr/sbin:/usr/bin:/sbin:/bin"
+
+__EOD
+
+sub gen_pve_vzdump_symlink {
+
+    my $filename = "/etc/pve/vzdump.cron";
+
+    my $link_fn = "/etc/cron.d/vzdump";
+
+    if ((-f $filename) && (! -l $link_fn)) {
+	rename($link_fn, "/root/etc_cron_vzdump.org"); # make backup if file exists
+	symlink($filename, $link_fn);
+    }
+}
+
+sub gen_pve_vzdump_files {
+
+    my $filename = "/etc/pve/vzdump.cron";
+
+    PVE::Tools::file_set_contents($filename, $vzdump_cron_dummy)
+	if ! -f $filename;
+
+    gen_pve_vzdump_symlink();
+};
+
+# join helpers
+
+sub assert_joinable {
+    my ($local_addr, $link0, $link1, $force) = @_;
+
+    my $errors = '';
+    my $error = sub { $errors .= "* $_[0]\n"; };
+
+    if (-f $authfile) {
+	$error->("authentication key '$authfile' already exists");
+    }
+
+    if (-f $clusterconf)  {
+	$error->("cluster config '$clusterconf' already exists");
+    }
+
+    my $vmlist = PVE::Cluster::get_vmlist();
+    if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
+	$error->("this host already contains virtual guests");
+    }
+
+    if (PVE::Tools::run_command(['corosync-quorumtool', '-l'], noerr => 1, quiet => 1) == 0) {
+	$error->("corosync is already running, is this node already in a cluster?!");
+    }
+
+    # check if corosync ring IPs are configured on the current nodes interfaces
+    my $check_ip = sub {
+	my $ip = shift // return;
+	my $logid = shift;
+	if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+	    my $host = $ip;
+	    eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+	    if ($@) {
+		$error->("$logid: cannot use '$host': $@\n") ;
+		return;
+	    }
+	}
+
+	my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+	my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+	$error->("$logid: cannot use IP '$ip', it must be configured exactly once on local node!\n")
+	    if (scalar(@$configured_ips) != 1);
+    };
+
+    $check_ip->($local_addr, 'local node address');
+    $check_ip->($link0->{address}, 'ring0') if defined($link0);
+    $check_ip->($link1->{address}, 'ring1') if defined($link1);
+
+    if ($errors) {
+	warn "detected the following error(s):\n$errors";
+	die "Check if node may join a cluster failed!\n" if !$force;
+    }
+}
+
+sub join {
+    my ($param) = @_;
+
+    my $nodename = PVE::INotify::nodename();
+    my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
+
+    my $link0 = PVE::Cluster::parse_corosync_link($param->{link0});
+    my $link1 = PVE::Cluster::parse_corosync_link($param->{link1});
+
+    # check if we can join with the given parameters and current node state
+    assert_joinable($local_ip_address, $link0, $link1, $param->{force});
+
+    setup_sshd_config();
+    setup_rootsshconfig();
+    setup_ssh_keys();
+
+    # make sure known_hosts is on local filesystem
+    ssh_unmerge_known_hosts();
+
+    my $host = $param->{hostname};
+    my $conn_args = {
+	username => 'root at pam',
+	password => $param->{password},
+	cookie_name => 'PVEAuthCookie',
+	protocol => 'https',
+	host => $host,
+	port => 8006,
+    };
+
+    if (my $fp = $param->{fingerprint}) {
+	$conn_args->{cached_fingerprints} = { uc($fp) => 1 };
+    } else {
+	# API schema ensures that we can only get here from CLI handler
+	$conn_args->{manual_verification} = 1;
+    }
+
+    print "Establishing API connection with host '$host'\n";
+
+    my $conn = PVE::APIClient::LWP->new(%$conn_args);
+    $conn->login();
+
+    # login raises an exception on failure, so if we get here we're good
+    print "Login succeeded.\n";
+
+    my $args = {};
+    $args->{force} = $param->{force} if defined($param->{force});
+    $args->{nodeid} = $param->{nodeid} if $param->{nodeid};
+    $args->{votes} = $param->{votes} if defined($param->{votes});
+    # just pass the un-parsed string through, or as we've address as the
+    # default_key, we can just pass the fallback directly too
+    $args->{link0} = $param->{link0} // $local_ip_address;
+    $args->{link1} = $param->{link1} if defined($param->{link1});
+
+    print "Request addition of this node\n";
+    my $res = $conn->post("/cluster/config/nodes/$nodename", $args);
+
+    print "Join request OK, finishing setup locally\n";
+
+    # added successfuly - now prepare local node
+    finish_join($nodename, $res->{corosync_conf}, $res->{corosync_authkey});
+}
+
+sub finish_join {
+    my ($nodename, $corosync_conf, $corosync_authkey) = @_;
+
+    mkdir "$localclusterdir";
+    PVE::Tools::file_set_contents($authfile, $corosync_authkey);
+    PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
+
+    print "stopping pve-cluster service\n";
+    my $cmd = ['systemctl', 'stop', 'pve-cluster'];
+    PVE::Tools::run_command($cmd, errmsg => "can't stop pve-cluster service");
+
+    my $dbfile = PVE::Cluster::cfs_backup_database();
+    unlink $dbfile;
+
+    $cmd = ['systemctl', 'start', 'corosync', 'pve-cluster'];
+    PVE::Tools::run_command($cmd, errmsg => "starting pve-cluster failed");
+
+    # wait for quorum
+    my $printqmsg = 1;
+    while (!PVE::Cluster::check_cfs_quorum(1)) {
+	if ($printqmsg) {
+	    print "waiting for quorum...";
+	    STDOUT->flush();
+	    $printqmsg = 0;
+	}
+	sleep(1);
+    }
+    print "OK\n" if !$printqmsg;
+
+    updatecerts_and_ssh(1);
+
+    print "generated new node certificate, restart pveproxy and pvedaemon services\n";
+    PVE::Tools::run_command(['systemctl', 'reload-or-restart', 'pvedaemon', 'pveproxy']);
+
+    print "successfully added node '$nodename' to cluster.\n";
+}
+
+sub updatecerts_and_ssh {
+    my ($force_new_cert, $silent) = @_;
+
+    my $p = sub { print "$_[0]\n" if !$silent };
+
+    setup_rootsshconfig();
+
+    gen_pve_vzdump_symlink();
+
+    if (!PVE::Cluster::check_cfs_quorum(1)) {
+	return undef if $silent;
+	die "no quorum - unable to update files\n";
+    }
+
+    setup_ssh_keys();
+
+    my $nodename = PVE::INotify::nodename();
+    my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
+
+    $p->("(re)generate node files");
+    $p->("generate new node certificate") if $force_new_cert;
+    gen_pve_node_files($nodename, $local_ip_address, $force_new_cert);
+
+    $p->("merge authorized SSH keys and known hosts");
+    ssh_merge_keys();
+    ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+    gen_pve_vzdump_files();
+}
+
+1;
-- 
2.20.1





More information about the pve-devel mailing list