[pve-devel] [PATCH v5 qemu-server 2/3] add QemuMigrateExternal.pm

David Limbeck d.limbeck at proxmox.com
Tue Feb 19 15:54:21 CET 2019


some comments inline.

On 1/29/19 2:20 AM, Alexandre Derumier wrote:
> ---
>   PVE/Makefile               |   1 +
>   PVE/QemuMigrateExternal.pm | 872 +++++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 873 insertions(+)
>   create mode 100644 PVE/QemuMigrateExternal.pm
>
> diff --git a/PVE/Makefile b/PVE/Makefile
> index 2c800f6..0494cfb 100644
> --- a/PVE/Makefile
> +++ b/PVE/Makefile
> @@ -1,6 +1,7 @@
>   PERLSOURCE = 			\
>   	QemuServer.pm		\
>   	QemuMigrate.pm		\
> +	QemuMigrateExternal.pm	\
>   	QMPClient.pm		\
>   	QemuConfig.pm
>   
> diff --git a/PVE/QemuMigrateExternal.pm b/PVE/QemuMigrateExternal.pm
> new file mode 100644
> index 0000000..ff1f46d
> --- /dev/null
> +++ b/PVE/QemuMigrateExternal.pm
> @@ -0,0 +1,872 @@
> +package PVE::QemuMigrateExternal;
> +
> +use strict;
> +use warnings;
> +use PVE::AbstractMigrate;
> +use IO::File;
> +use IPC::Open2;
> +use POSIX qw( WNOHANG );
> +use PVE::INotify;
> +use PVE::Tools;
> +use PVE::Cluster;
> +use PVE::Storage;
> +use PVE::QemuServer;
> +use Time::HiRes qw( usleep );
> +use PVE::RPCEnvironment;
> +use PVE::ReplicationConfig;
> +use PVE::ReplicationState;
> +use PVE::Replication;
> +use Storable qw(dclone);
> +
> +use base qw(PVE::AbstractMigrate);
> +
> +sub fork_command_pipe {
> +    my ($self, $cmd) = @_;
> +
> +    my $reader = IO::File->new();
> +    my $writer = IO::File->new();
> +
> +    my $orig_pid = $$;
> +
> +    my $cpid;
> +
> +    eval { $cpid = open2($reader, $writer, @$cmd); };
> +
> +    my $err = $@;
> +
> +    # catch exec errors
> +    if ($orig_pid != $$) {
> +	$self->log('err', "can't fork command pipe\n");
> +	POSIX::_exit(1);
> +	kill('KILL', $$);
> +    }
> +
> +    die $err if $err;
> +
> +    return { writer => $writer, reader => $reader, pid => $cpid };
> +}
> +
> +sub finish_command_pipe {
> +    my ($self, $cmdpipe, $timeout) = @_;
> +
> +    my $cpid = $cmdpipe->{pid};
> +    return if !defined($cpid);
> +
> +    my $writer = $cmdpipe->{writer};
> +    my $reader = $cmdpipe->{reader};
> +
> +    $writer->close();
> +    $reader->close();
> +
> +    my $collect_child_process = sub {
> +	my $res = waitpid($cpid, WNOHANG);
> +	if (defined($res) && ($res == $cpid)) {
> +	    delete $cmdpipe->{cpid};
> +	    return 1;
> +	} else {
> +	    return 0;
> +	}
> +     };
> +
> +    if ($timeout) {
> +	for (my $i = 0; $i < $timeout; $i++) {
> +	    return if &$collect_child_process();
> +	    sleep(1);
> +	}
> +    }
> +
> +    $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n");
> +    kill(15, $cpid);
> +
> +    # wait again
> +    for (my $i = 0; $i < 10; $i++) {
> +	return if &$collect_child_process();
> +	sleep(1);
> +    }
> +
> +    $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n");
> +    kill 9, $cpid;
> +    sleep 1;
> +
> +    $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n")
> +	if !&$collect_child_process();
> +}
> +
> +sub read_tunnel {
> +    my ($self, $tunnel, $timeout) = @_;
> +
> +    $timeout = 60 if !defined($timeout);
> +
> +    my $reader = $tunnel->{reader};
> +
> +    my $output;
> +    eval {
> +	PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; });
> +    };
> +    die "reading from tunnel failed: $@\n" if $@;
> +
> +    chomp $output;
> +
> +    return $output;
> +}
> +
> +sub write_tunnel {
> +    my ($self, $tunnel, $timeout, $command) = @_;
> +
> +    $timeout = 60 if !defined($timeout);
> +
> +    my $writer = $tunnel->{writer};
> +
> +    eval {
> +	PVE::Tools::run_with_timeout($timeout, sub {
> +	    print $writer "$command\n";
> +	    $writer->flush();
> +	});
> +    };
> +    die "writing to tunnel failed: $@\n" if $@;
> +
> +    if ($tunnel->{version} && $tunnel->{version} >= 1) {
> +	my $res = eval { $self->read_tunnel($tunnel, 10); };
> +	die "no reply to command '$command': $@\n" if $@;
> +
> +	if ($res eq 'OK') {
> +	    return;
> +	} else {
> +	    die "tunnel replied '$res' to command '$command'\n";
> +	}
> +    }
> +}
> +
> +sub fork_tunnel {
> +    my ($self, $tunnel_addr) = @_;
> +
> +    my @localtunnelinfo = defined($tunnel_addr) ? ('-L' , $tunnel_addr ) : ();
> +
> +    my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ];
> +
> +    my $tunnel = $self->fork_command_pipe($cmd);
> +
> +    eval {
> +	my $helo = $self->read_tunnel($tunnel, 60);
> +	die "no reply\n" if !$helo;
> +	die "no quorum on target node\n" if $helo =~ m/^no quorum$/;
> +	die "got strange reply from mtunnel ('$helo')\n"
> +	    if $helo !~ m/^tunnel online$/;
> +    };
> +    my $err = $@;
> +
> +    eval {
> +	my $ver = $self->read_tunnel($tunnel, 10);
> +	if ($ver =~ /^ver (\d+)$/) {
> +	    $tunnel->{version} = $1;
> +	    $self->log('info', "ssh tunnel $ver\n");
> +	} else {
> +	    $err = "received invalid tunnel version string '$ver'\n" if !$err;
> +	}
> +    };
> +
> +    if ($err) {
> +	$self->finish_command_pipe($tunnel);
> +	die "can't open migration tunnel - $err";
> +    }
> +    return $tunnel;
> +}
> +
> +sub finish_tunnel {
> +    my ($self, $tunnel) = @_;
> +
> +    eval { $self->write_tunnel($tunnel, 30, 'quit'); };
> +    my $err = $@;
> +
> +    $self->finish_command_pipe($tunnel, 30);
> +
> +    if ($tunnel->{sock_addr}) {
> +	# ssh does not clean up on local host
> +	my $cmd = ['rm', '-f', $tunnel->{sock_addr}]; #
> +	PVE::Tools::run_command($cmd);
> +
> +	# .. and just to be sure check on remote side
> +	unshift @{$cmd}, @{$self->{rem_ssh}};
> +	PVE::Tools::run_command($cmd);
> +    }
> +
> +    die $err if $err;
> +}
> +
> +sub lock_vm {
> +    my ($self, $vmid, $code, @param) = @_;
> +
> +    return PVE::QemuConfig->lock_config($vmid, $code, @param);
> +}
> +
> +sub prepare {
> +    my ($self, $vmid) = @_;
> +
> +    my $online = $self->{opts}->{online};
> +
> +    $self->{storecfg} = PVE::Storage::config();
> +
> +    # test if VM exists
> +    my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid);
> +
> +    PVE::QemuConfig->check_lock($conf);
> +
> +    my $running = 0;
> +    if (my $pid = PVE::QemuServer::check_running($vmid)) {
> +	die "can't migrate running VM without --online\n" if !$online;
> +	$running = $pid;
> +
> +	$self->{forcemachine} = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
> +
> +    }
> +
> +    if (my $loc_res = PVE::QemuServer::check_local_resources($conf, 1)) {
> +	if ($self->{running} || !$self->{opts}->{force}) {
> +	    die "can't migrate VM which uses local devices\n";
> +	} else {
> +	    $self->log('info', "migrating VM which uses local devices");
> +	}
> +    }
> +
> +    # test ssh connection
> +    my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
> +    eval { $self->cmd_quiet($cmd); };
> +    die "Can't connect to destination address using public key\n" if $@;
> +
> +
> +    push @{$self->{rem_ssh}}, '-i', $self->{opts}->{migration_external_sshkey};
The identity file should be used for the connection test as well.

> +
> +    # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
> +    # instead we pipe it through STDIN
> +    PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
> +	my $line = shift;
> +
> +	if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
> +	    $raddr = $1;
> +	    $rport = int($2);
> +	    $ruri = "tcp:$raddr:$rport";
> +	}
> +	elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
> +	    $raddr = $1;
> +	    die "Destination UNIX sockets VMID does not match source VMID" if $targetvmid ne $2;
> +	    $ruri = "unix:$raddr";
> +	}
> +	elsif ($line =~ m/^migration listens on port (\d+)$/) {
> +	    $raddr = "localhost";
> +	    $rport = int($1);
> +	    $ruri = "tcp:$raddr:$rport";
> +	}
> +        elsif ($line =~ m/^spice listens on port (\d+)$/) {
> +	    $spice_port = int($1);
> +	}
> +        elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
> +	    my $volid = $4;
> +	    my $nbd_uri = "nbd:$1:$2:exportname=$3";
> +	    my $targetdrive = $3;
> +	    $targetdrive =~ s/drive-//g;
> +
> +	    $self->{target_drive}->{$targetdrive}->{volid} = $volid;
> +	    $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
> +
> +	}
wrong indentation for those 2 'elsif
> +    }, errfunc => sub {
> +	my $line = shift;
> +	$self->log('info', $line);
> +    });
> +
>
> +
> +    if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +	my $authuser = $rpcenv->get_user();
> +
> +	my (undef, $proxyticket) = PVE::AccessControl::assemble_spice_ticket($authuser, $vmid, $self->{node});
> +
> +	my $filename = "/etc/pve/nodes/$self->{node}/pve-ssl.pem";
> +        my $subject =  PVE::AccessControl::read_x509_subject_spice($filename);
wrong indentation.
> +
> +	$self->log('info', "spice client_migrate_info");
> +
> +	eval {
> +	    PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "client_migrate_info", protocol => 'spice',
> +						hostname => $proxyticket, 'tls-port' => $spice_port,
> +						'cert-subject' => $subject);
> +	};
> +	$self->log('info', "client_migrate_info error: $@") if $@;
> +
> +    }
> +



More information about the pve-devel mailing list