[pve-devel] [PATCH v5 qemu-server 2/3] add QemuMigrateExternal.pm
David Limbeck
d.limbeck at proxmox.com
Tue Feb 19 15:54:21 CET 2019
some comments inline.
On 1/29/19 2:20 AM, Alexandre Derumier wrote:
> ---
> PVE/Makefile | 1 +
> PVE/QemuMigrateExternal.pm | 872 +++++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 873 insertions(+)
> create mode 100644 PVE/QemuMigrateExternal.pm
>
> diff --git a/PVE/Makefile b/PVE/Makefile
> index 2c800f6..0494cfb 100644
> --- a/PVE/Makefile
> +++ b/PVE/Makefile
> @@ -1,6 +1,7 @@
> PERLSOURCE = \
> QemuServer.pm \
> QemuMigrate.pm \
> + QemuMigrateExternal.pm \
> QMPClient.pm \
> QemuConfig.pm
>
> diff --git a/PVE/QemuMigrateExternal.pm b/PVE/QemuMigrateExternal.pm
> new file mode 100644
> index 0000000..ff1f46d
> --- /dev/null
> +++ b/PVE/QemuMigrateExternal.pm
> @@ -0,0 +1,872 @@
> +package PVE::QemuMigrateExternal;
> +
> +use strict;
> +use warnings;
> +use PVE::AbstractMigrate;
> +use IO::File;
> +use IPC::Open2;
> +use POSIX qw( WNOHANG );
> +use PVE::INotify;
> +use PVE::Tools;
> +use PVE::Cluster;
> +use PVE::Storage;
> +use PVE::QemuServer;
> +use Time::HiRes qw( usleep );
> +use PVE::RPCEnvironment;
> +use PVE::ReplicationConfig;
> +use PVE::ReplicationState;
> +use PVE::Replication;
> +use Storable qw(dclone);
> +
> +use base qw(PVE::AbstractMigrate);
> +
> +sub fork_command_pipe {
> + my ($self, $cmd) = @_;
> +
> + my $reader = IO::File->new();
> + my $writer = IO::File->new();
> +
> + my $orig_pid = $$;
> +
> + my $cpid;
> +
> + eval { $cpid = open2($reader, $writer, @$cmd); };
> +
> + my $err = $@;
> +
> + # catch exec errors
> + if ($orig_pid != $$) {
> + $self->log('err', "can't fork command pipe\n");
> + POSIX::_exit(1);
> + kill('KILL', $$);
> + }
> +
> + die $err if $err;
> +
> + return { writer => $writer, reader => $reader, pid => $cpid };
> +}
> +
> +sub finish_command_pipe {
> + my ($self, $cmdpipe, $timeout) = @_;
> +
> + my $cpid = $cmdpipe->{pid};
> + return if !defined($cpid);
> +
> + my $writer = $cmdpipe->{writer};
> + my $reader = $cmdpipe->{reader};
> +
> + $writer->close();
> + $reader->close();
> +
> + my $collect_child_process = sub {
> + my $res = waitpid($cpid, WNOHANG);
> + if (defined($res) && ($res == $cpid)) {
> + delete $cmdpipe->{cpid};
> + return 1;
> + } else {
> + return 0;
> + }
> + };
> +
> + if ($timeout) {
> + for (my $i = 0; $i < $timeout; $i++) {
> + return if &$collect_child_process();
> + sleep(1);
> + }
> + }
> +
> + $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n");
> + kill(15, $cpid);
> +
> + # wait again
> + for (my $i = 0; $i < 10; $i++) {
> + return if &$collect_child_process();
> + sleep(1);
> + }
> +
> + $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n");
> + kill 9, $cpid;
> + sleep 1;
> +
> + $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n")
> + if !&$collect_child_process();
> +}
> +
> +sub read_tunnel {
> + my ($self, $tunnel, $timeout) = @_;
> +
> + $timeout = 60 if !defined($timeout);
> +
> + my $reader = $tunnel->{reader};
> +
> + my $output;
> + eval {
> + PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; });
> + };
> + die "reading from tunnel failed: $@\n" if $@;
> +
> + chomp $output;
> +
> + return $output;
> +}
> +
> +sub write_tunnel {
> + my ($self, $tunnel, $timeout, $command) = @_;
> +
> + $timeout = 60 if !defined($timeout);
> +
> + my $writer = $tunnel->{writer};
> +
> + eval {
> + PVE::Tools::run_with_timeout($timeout, sub {
> + print $writer "$command\n";
> + $writer->flush();
> + });
> + };
> + die "writing to tunnel failed: $@\n" if $@;
> +
> + if ($tunnel->{version} && $tunnel->{version} >= 1) {
> + my $res = eval { $self->read_tunnel($tunnel, 10); };
> + die "no reply to command '$command': $@\n" if $@;
> +
> + if ($res eq 'OK') {
> + return;
> + } else {
> + die "tunnel replied '$res' to command '$command'\n";
> + }
> + }
> +}
> +
> +sub fork_tunnel {
> + my ($self, $tunnel_addr) = @_;
> +
> + my @localtunnelinfo = defined($tunnel_addr) ? ('-L' , $tunnel_addr ) : ();
> +
> + my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ];
> +
> + my $tunnel = $self->fork_command_pipe($cmd);
> +
> + eval {
> + my $helo = $self->read_tunnel($tunnel, 60);
> + die "no reply\n" if !$helo;
> + die "no quorum on target node\n" if $helo =~ m/^no quorum$/;
> + die "got strange reply from mtunnel ('$helo')\n"
> + if $helo !~ m/^tunnel online$/;
> + };
> + my $err = $@;
> +
> + eval {
> + my $ver = $self->read_tunnel($tunnel, 10);
> + if ($ver =~ /^ver (\d+)$/) {
> + $tunnel->{version} = $1;
> + $self->log('info', "ssh tunnel $ver\n");
> + } else {
> + $err = "received invalid tunnel version string '$ver'\n" if !$err;
> + }
> + };
> +
> + if ($err) {
> + $self->finish_command_pipe($tunnel);
> + die "can't open migration tunnel - $err";
> + }
> + return $tunnel;
> +}
> +
> +sub finish_tunnel {
> + my ($self, $tunnel) = @_;
> +
> + eval { $self->write_tunnel($tunnel, 30, 'quit'); };
> + my $err = $@;
> +
> + $self->finish_command_pipe($tunnel, 30);
> +
> + if ($tunnel->{sock_addr}) {
> + # ssh does not clean up on local host
> + my $cmd = ['rm', '-f', $tunnel->{sock_addr}]; #
> + PVE::Tools::run_command($cmd);
> +
> + # .. and just to be sure check on remote side
> + unshift @{$cmd}, @{$self->{rem_ssh}};
> + PVE::Tools::run_command($cmd);
> + }
> +
> + die $err if $err;
> +}
> +
> +sub lock_vm {
> + my ($self, $vmid, $code, @param) = @_;
> +
> + return PVE::QemuConfig->lock_config($vmid, $code, @param);
> +}
> +
> +sub prepare {
> + my ($self, $vmid) = @_;
> +
> + my $online = $self->{opts}->{online};
> +
> + $self->{storecfg} = PVE::Storage::config();
> +
> + # test if VM exists
> + my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid);
> +
> + PVE::QemuConfig->check_lock($conf);
> +
> + my $running = 0;
> + if (my $pid = PVE::QemuServer::check_running($vmid)) {
> + die "can't migrate running VM without --online\n" if !$online;
> + $running = $pid;
> +
> + $self->{forcemachine} = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
> +
> + }
> +
> + if (my $loc_res = PVE::QemuServer::check_local_resources($conf, 1)) {
> + if ($self->{running} || !$self->{opts}->{force}) {
> + die "can't migrate VM which uses local devices\n";
> + } else {
> + $self->log('info', "migrating VM which uses local devices");
> + }
> + }
> +
> + # test ssh connection
> + my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
> + eval { $self->cmd_quiet($cmd); };
> + die "Can't connect to destination address using public key\n" if $@;
> +
> +
> + push @{$self->{rem_ssh}}, '-i', $self->{opts}->{migration_external_sshkey};
The identity file should be used for the connection test as well.
> +
> + # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
> + # instead we pipe it through STDIN
> + PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
> + my $line = shift;
> +
> + if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
> + $raddr = $1;
> + $rport = int($2);
> + $ruri = "tcp:$raddr:$rport";
> + }
> + elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
> + $raddr = $1;
> + die "Destination UNIX sockets VMID does not match source VMID" if $targetvmid ne $2;
> + $ruri = "unix:$raddr";
> + }
> + elsif ($line =~ m/^migration listens on port (\d+)$/) {
> + $raddr = "localhost";
> + $rport = int($1);
> + $ruri = "tcp:$raddr:$rport";
> + }
> + elsif ($line =~ m/^spice listens on port (\d+)$/) {
> + $spice_port = int($1);
> + }
> + elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
> + my $volid = $4;
> + my $nbd_uri = "nbd:$1:$2:exportname=$3";
> + my $targetdrive = $3;
> + $targetdrive =~ s/drive-//g;
> +
> + $self->{target_drive}->{$targetdrive}->{volid} = $volid;
> + $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
> +
> + }
wrong indentation for those 2 'elsif
> + }, errfunc => sub {
> + my $line = shift;
> + $self->log('info', $line);
> + });
> +
>
> +
> + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
> + my $rpcenv = PVE::RPCEnvironment::get();
> + my $authuser = $rpcenv->get_user();
> +
> + my (undef, $proxyticket) = PVE::AccessControl::assemble_spice_ticket($authuser, $vmid, $self->{node});
> +
> + my $filename = "/etc/pve/nodes/$self->{node}/pve-ssl.pem";
> + my $subject = PVE::AccessControl::read_x509_subject_spice($filename);
wrong indentation.
> +
> + $self->log('info', "spice client_migrate_info");
> +
> + eval {
> + PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "client_migrate_info", protocol => 'spice',
> + hostname => $proxyticket, 'tls-port' => $spice_port,
> + 'cert-subject' => $subject);
> + };
> + $self->log('info', "client_migrate_info error: $@") if $@;
> +
> + }
> +
More information about the pve-devel
mailing list