[pve-devel] [PATCH qemu-server 4/4] implement PoC migration to remote cluster/node

Fabian Grünbichler f.gruenbichler at proxmox.com
Fri Mar 6 11:20:35 CET 2020


there's obviously lots of TODOs and FIXMEs in here, the big ones are:
- better handling of storage switching
- handling of network switching
- implementing cleanup
- actually dropping the local/source config and disks
- NBD client side is kept open by Qemu, so we need to terminate the data
tunnel in order to stop the VM

I think it would make sense to re-factor vm_start and pass in all the
(mostly) migration-related parameters as a hash. that way, we could tell
the remote side that this is a remote incoming migration, and split out
the disk/network updating into its own step in mtunnel that can happen
before, and then start NBD for all disks, not just local, newly
allocated ones.

for mapping disks/NICs I see three options:
- global switch to map everything to one target, with fall-back to keep
as-is (current)
- mapping of source storage/bridge to target storage/bridge
- mapping of source disks (volid)/NIC to target storage/bridge

the intention of sending this in the current state is mainly to get
feedback early on on whether we want to go down this route, or use a
totally different approach.

Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
 PVE/QemuMigrate.pm | 510 +++++++++++++++++++++++++++++++++------------
 1 file changed, 376 insertions(+), 134 deletions(-)

diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm
index 6db5e62..01a5b12 100644
--- a/PVE/QemuMigrate.pm
+++ b/PVE/QemuMigrate.pm
@@ -3,9 +3,18 @@ package PVE::QemuMigrate;
 use strict;
 use warnings;
 use PVE::AbstractMigrate;
+
+use Data::Dumper;
+
 use IO::File;
+use IO::Socket::UNIX;
 use IPC::Open2;
+use JSON;
+use MIME::Base64;
 use POSIX qw( WNOHANG );
+use URI::Escape;
+
+use PVE::APIClient::LWP;
 use PVE::INotify;
 use PVE::Tools;
 use PVE::Cluster;
@@ -15,9 +24,11 @@ use PVE::QemuServer::Machine;
 use PVE::QemuServer::Monitor qw(mon_cmd);
 use Time::HiRes qw( usleep );
 use PVE::RPCEnvironment;
+use PVE::RemoteConfig;
 use PVE::ReplicationConfig;
 use PVE::ReplicationState;
 use PVE::Replication;
+use PVE::WebSocket;
 
 use base qw(PVE::AbstractMigrate);
 
@@ -56,8 +67,8 @@ sub finish_command_pipe {
     my $writer = $cmdpipe->{writer};
     my $reader = $cmdpipe->{reader};
 
-    $writer->close();
-    $reader->close();
+    $writer->close() if defined($writer);
+    $reader->close() if defined($reader);
 
     my $collect_child_process = sub {
 	my $res = waitpid($cpid, WNOHANG);
@@ -173,6 +184,109 @@ sub fork_tunnel {
     return $tunnel;
 }
 
+sub fork_websocket_tunnel {
+    my ($self) = @_;
+
+    my $remote = $self->{opts}->{remote};
+    my $remote_config = PVE::RemoteConfig->new();
+    my ($ips, $conn_args) = $remote_config->get_remote_info($self->{opts}->{remote}, $self->{node});
+    my $conn = PVE::APIClient::LWP->new(%$conn_args);
+
+    my $mtunnel_worker = $conn->post("api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnel", { version => 2 });
+    $self->log('info', "Spawned migration tunnel worker on '$self->{opts}->{remote}/$self->{node}'");
+    # FIXME add mtunnel ticket
+    my $api_path = "/api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnelwebsocket?socket=".uri_escape("/run/qemu-server/$self->{vmid}.mtunnel");
+
+    # TODO use migration network?
+    my $ws = PVE::WebSocket->new($conn->{host}, 8006, $api_path);
+
+    my $auth_header = "Authorization: $conn->{apitoken}";
+
+    $ws->connect($auth_header);
+
+    my $reader = IO::Pipe->new();
+    my $writer = IO::Pipe->new();
+
+    my $cpid = fork();
+    if ($cpid) {
+	my $tunnel = { writer => $writer->writer(), reader => $reader->reader(), socket => $ws, pid => $cpid, version => 2 };
+	# FIXME: read version
+	print "hello: '".$self->read_tunnel($tunnel, 10)."'\n";
+	print "hello: '".$self->read_tunnel($tunnel, 10)."'\n";
+	$self->{api_conn} = $conn;
+	$self->{data_tunnels} = [];
+	return $tunnel;
+    } else {
+	$ws->{reader} = $writer->reader();
+	$ws->{writer} = $reader->writer();
+	$ws->process();
+	exit 0;
+    }
+}
+
+sub fork_websocket_data_tunnel {
+    my ($self, $local, $remote) = @_;
+    # TODO implement TCP?
+
+    unlink $local;
+    my $local_socket = IO::Socket::UNIX->new(
+	Type => SOCK_STREAM(),
+	Local => $local,
+	Listen => 5,
+    );
+
+    die "could not bind to local socket '$local' of data tunnel  - $!\n"
+	if !$local_socket;
+
+    my $cpid = fork();
+    if ($cpid) {
+	$local_socket->close();
+	return { local => $local, pid => $cpid };
+    } else {
+	$self->log('info', "forked websocket data tunnel $local!");
+	my $clients = [];
+	my $exit_handler = sub {
+	    $self->log('info', "Closing data tunnel '$local'");
+	    $local_socket->close();
+	    foreach my $c (@$clients) {
+		$self->log('info', "closing client socket for $local #$c->{index}");
+		$self->finish_command_pipe($c);
+	    }
+	    $self->log('info', "Closed data tunnel '$local'");
+	};
+	local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = $exit_handler;
+	my $index = 0;
+	while (my $client = $local_socket->accept()) {
+	    $index++;
+	    $self->log('info', "accepted new connection #$index on $local, establishing web socket connection..\n");
+	    $cpid = fork();
+	    if ($cpid) {
+		$client->close();
+		push @$clients, { pid => $cpid, index => $index };
+	    } else {
+		$local_socket->close();
+		local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = 'IGNORE';
+		my $conn = $self->{api_conn};
+		my $api_path = "/api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnelwebsocket?socket=".uri_escape($remote);
+		my $ws = PVE::WebSocket->new($conn->{host}, 8006, $api_path);
+
+		my $auth_header = "Authorization: $conn->{apitoken}";
+
+		$ws->connect($auth_header);
+		local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub { $ws->close() };
+
+		$self->log('info', "connected websocket data tunnel $local #$index");
+		$ws->{reader} = $client;
+		$ws->{writer} = $client;
+		$ws->process();
+		$self->log('info', "processing finished on websocket data tunnel $local #$index..");
+		exit 0;
+	    }
+	}
+	exit 0;
+    }
+}
+
 sub finish_tunnel {
     my ($self, $tunnel) = @_;
 
@@ -239,6 +353,12 @@ sub prepare {
 	my $targetsid = $self->{opts}->{targetstorage} // $sid;
 
 	my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
+
+	if ($self->{opts}->{remote}) {
+	    push @$need_activate, $volid;
+	    next;
+	}
+
 	PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
 
 	if ($scfg->{shared}) {
@@ -256,10 +376,16 @@ sub prepare {
     # activate volumes
     PVE::Storage::activate_volumes($self->{storecfg}, $need_activate);
 
-    # test ssh connection
-    my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
-    eval { $self->cmd_quiet($cmd); };
-    die "Can't connect to destination address using public key\n" if $@;
+    if ($self->{opts}->{remote}) {
+	# test web socket connection
+	$self->{tunnel} = $self->fork_websocket_tunnel();
+	print "websocket tunnel started\n";
+    } else {
+	# test ssh connection
+	my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
+	eval { $self->cmd_quiet($cmd); };
+	die "Can't connect to destination address using public key\n" if $@;
+    }
 
     return $running;
 }
@@ -296,7 +422,7 @@ sub sync_disks {
 	my @sids = PVE::Storage::storage_ids($self->{storecfg});
 	foreach my $storeid (@sids) {
 	    my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
-	    next if $scfg->{shared};
+	    next if $scfg->{shared} && !$self->{opts}->{remote};
 	    next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1);
 
 	    # get list from PVE::Storage (for unused volumes)
@@ -307,7 +433,8 @@ sub sync_disks {
 	    my $targetsid = $override_targetsid // $storeid;
 
 	    # check if storage is available on target node
-	    PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
+	    PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node})
+		if !$self->{opts}->{remote};
 
 	    PVE::Storage::foreach_volid($dl, sub {
 		my ($volid, $sid, $volname) = @_;
@@ -345,9 +472,10 @@ sub sync_disks {
 	    my $targetsid = $override_targetsid // $sid;
 	    # check if storage is available on both nodes
 	    my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
-	    PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
+	    PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node})
+		if !$self->{opts}->{remote};
 
-	    return if $scfg->{shared};
+	    return if $scfg->{shared} && !$self->{opts}->{remote};
 
 	    $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot';
 
@@ -423,6 +551,9 @@ sub sync_disks {
 
 	    my $migratable = $scfg->{type} =~ /^(?:dir|zfspool|lvmthin|lvm)$/;
 
+	    # TODO: implement properly
+	    $migratable = 1 if $self->{opts}->{remote};
+
 	    die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n"
 		if !$migratable;
 
@@ -435,6 +566,9 @@ sub sync_disks {
 	my $rep_cfg = PVE::ReplicationConfig->new();
 	if (my $jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node})) {
 	    die "can't live migrate VM with replicated volumes\n" if $self->{running};
+	    die "can't migrate VM with replicated volumes to remote cluster/node\n"
+		if $self->{opts}->{remote};
+
 	    $self->log('info', "replicating disk images");
 	    my $start_time = time();
 	    my $logfunc = sub { $self->log('info', shift) };
@@ -469,6 +603,9 @@ sub sync_disks {
 		next;
 	    } else {
 		next if $self->{replicated_volumes}->{$volid};
+		die "storage migration to remote cluster/node not implemented yet\n"
+		    if $self->{opts}->{remote};
+
 		push @{$self->{volumes}}, $volid;
 		my $opts = $self->{opts};
 		my $insecure = $opts->{migration_type} eq 'insecure';
@@ -489,8 +626,12 @@ sub sync_disks {
 sub cleanup_remotedisks {
     my ($self) = @_;
 
-    foreach my $target_drive (keys %{$self->{target_drive}}) {
+    if ($self->{opts}->{remote}) {
+	warn "cleanup not yet implemented";
+	return;
+    }
 
+    foreach my $target_drive (keys %{$self->{target_drive}}) {
 	my $drive = PVE::QemuServer::parse_drive($target_drive, $self->{target_drive}->{$target_drive}->{drivestr});
 	my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
 
@@ -520,6 +661,32 @@ sub phase1 {
     # sync_disks fixes disk sizes to match their actual size, write changes so
     # target allocates correct volumes
     PVE::QemuConfig->write_config($vmid, $conf);
+
+    if ($self->{opts}->{remote}) {
+	# TODO without disks here, send disks + nics as separate commands?
+	# we want single line here!
+	my $remote_conf = PVE::QemuConfig->load_config($vmid);
+	if (my $targetsid = $self->{opts}->{targetstorage}) {
+	    PVE::QemuServer::foreach_drive($remote_conf, sub {
+		my ($ds, $drive) = @_;
+
+		return if PVE::QemuServer::drive_is_cdrom($drive);
+
+		my $volid = $drive->{file};
+		return if !$volid;
+		return if ! grep { $_ eq $volid} @{$self->{online_local_volumes}};
+
+		$self->log('info', "Rewriting config to move '$ds' - '$volid' to '$targetsid'");
+
+		my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+		$drive->{file} = "$targetsid:$volname";
+		$remote_conf->{$ds} = PVE::QemuServer::print_drive($drive);
+	    });
+	}
+	my $conf_str = MIME::Base64::encode_base64url(JSON::encode_json($remote_conf));
+	$self->write_tunnel($self->{tunnel}, 10, "config $conf_str");
+    }
+
 };
 
 sub phase1_cleanup {
@@ -549,13 +716,9 @@ sub phase2 {
 
     $self->log('info', "starting VM $vmid on remote node '$self->{node}'");
 
-    my $raddr;
-    my $rport;
-    my $ruri; # the whole migration dst. URI (protocol:address[:port])
-    my $nodename = PVE::INotify::nodename();
+    my $info = {};
 
-    ## start on remote node
-    my $cmd = [@{$self->{rem_ssh}}];
+    my $nodename = PVE::INotify::nodename();
 
     my $spice_ticket;
     if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
@@ -563,113 +726,145 @@ sub phase2 {
 	$spice_ticket = $res->{ticket};
     }
 
-    push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename;
-
     my $migration_type = $self->{opts}->{migration_type};
+    my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix';
 
-    push @$cmd, '--migration_type', $migration_type;
-
-    push @$cmd, '--migration_network', $self->{opts}->{migration_network}
-      if $self->{opts}->{migration_network};
-
-    if ($migration_type eq 'insecure') {
-	push @$cmd, '--stateuri', 'tcp';
+    ## start on remote node
+    if ($self->{opts}->{remote}) {
+	die "insecure migration to remote cluster/node not implemented yet\n"
+	    if $migration_type eq 'insecure';
+
+	my $start_json = JSON::encode_json({
+	    spice => $spice_ticket,
+	    migration_type => $migration_type,
+	    state_uri => $state_uri,
+	    network => $self->{opts}->{migration_network},
+	    machine => $self->{forcemachine},
+	    targetstorage => $self->{opts}->{targetstorage},
+	});
+	$self->write_tunnel($self->{tunnel}, 10, "start $start_json");
+	$info = JSON::decode_json($self->read_tunnel($self->{tunnel}, 10));
+	print Dumper($info), "\n";
     } else {
-	push @$cmd, '--stateuri', 'unix';
-    }
+	my $cmd = [@{$self->{rem_ssh}}];
+	push @$cmd , 'qm', 'start', $vmid;
+	push @$cmd, '--skiplock', '--migratedfrom', $nodename;
+	push @$cmd, '--migration_type', $migration_type;
+	push @$cmd, '--migration_network', $self->{opts}->{migration_network}
+	    if $self->{opts}->{migration_network};
 
-    if ($self->{forcemachine}) {
-	push @$cmd, '--machine', $self->{forcemachine};
-    }
+	push @$cmd, '--stateuri', $state_uri;
 
-    if ($self->{online_local_volumes}) {
-	push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
-    }
+	if ($self->{forcemachine}) {
+	    push @$cmd, '--machine', $self->{forcemachine};
+	}
 
-    my $spice_port;
+	if ($self->{online_local_volumes}) {
+	    push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
+	}
 
-    # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
-    # instead we pipe it through STDIN
-    my $exitcode = PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
-	my $line = shift;
+	# Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
+	# instead we pipe it through STDIN
+	my $exitcode = PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
+	    my $line = shift;
 
-	if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
-	    $raddr = $1;
-	    $rport = int($2);
-	    $ruri = "tcp:$raddr:$rport";
-	}
-	elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
-	    $raddr = $1;
-	    die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
-	    $ruri = "unix:$raddr";
-	}
-	elsif ($line =~ m/^migration listens on port (\d+)$/) {
-	    $raddr = "localhost";
-	    $rport = int($1);
-	    $ruri = "tcp:$raddr:$rport";
-	}
-	elsif ($line =~ m/^spice listens on port (\d+)$/) {
-	    $spice_port = int($1);
-	}
-	elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
-	    my $drivestr = $4;
-	    my $nbd_uri = "nbd:$1:$2:exportname=$3";
-	    my $targetdrive = $3;
-	    $targetdrive =~ s/drive-//g;
+	    if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
+		$info->{raddr} = $1;
+		$info->{rport} = int($2);
+		$info->{ruri} = "tcp:$info->{raddr}:$info->{rport}";
+	    }
+	    elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
+		$info->{raddr} = $1;
+		die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
+		$info->{ruri} = "unix:$info->{raddr}";
+	    }
+	    elsif ($line =~ m/^migration listens on port (\d+)$/) {
+		$info->{raddr} = "localhost";
+		$info->{rport} = int($1);
+		$info->{ruri} = "tcp:$info->{raddr}:$info->{rport}";
+	    }
+	    elsif ($line =~ m/^spice listens on port (\d+)$/) {
+		$info->{spice_port} = int($1);
+	    }
+	    elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
+		my $drivestr = $4;
+		my $nbd_uri = "nbd:$1:$2:exportname=$3";
+		my $targetdrive = $3;
+		$targetdrive =~ s/drive-//g;
 
-	    $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr;
-	    $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
+		$info->{drives}->{$targetdrive}->{drivestr} = $drivestr;
+		$info->{drives}->{$targetdrive}->{nbd_uri} = $nbd_uri;
 
-	} elsif ($line =~ m/^QEMU: (.*)$/) {
-	    $self->log('info', "[$self->{node}] $1\n");
-	}
-    }, errfunc => sub {
-	my $line = shift;
-	$self->log('info', "[$self->{node}] $line");
-    }, noerr => 1);
+	    } elsif ($line =~ m/^QEMU: (.*)$/) {
+		$self->log('info', "[$self->{node}] $1\n");
+	    }
+	}, errfunc => sub {
+	    my $line = shift;
+	    $self->log('info', "[$self->{node}] $line");
+	}, noerr => 1);
 
-    die "remote command failed with exit code $exitcode\n" if $exitcode;
+	die "remote command failed with exit code $exitcode\n" if $exitcode;
+    }
 
-    die "unable to detect remote migration address\n" if !$raddr;
+    die "unable to detect remote migration address\n" if !$info->{raddr};
 
-    $self->log('info', "start remote tunnel");
+    foreach my $drive (keys %{$info->{drives}}) {
+	$self->{target_drive}->{$drive}->{drivestr} = $info->{drives}->{$drive}->{drivestr};
+	$self->{target_drive}->{$drive}->{nbd_uri} = $info->{drives}->{$drive}->{nbd_uri};
+    }
 
-    if ($migration_type eq 'secure') {
+    if ($self->{tunnel} && $self->{opts}->{remote}) {
+	# TODO support TCP?
+	if ($info->{ruri} =~ /^unix:/) {
+	    $self->log('info', "starting memory migration tunnel '$info->{raddr}' => '$info->{raddr}'");
+	    push @{$self->{data_tunnels}}, $self->fork_websocket_data_tunnel($info->{raddr}, $info->{raddr});
+	} else {
+	    die "unsupported migration uri '$info->{ruri}'\n";
+	}
+	if ($info->{nbd}) {
+	    $self->log('info', "starting local disk migration tunnel '$info->{nbd}' => '$info->{nbd}'");
+	    # keep open - we might have multiple disks!
+	    push @{$self->{data_tunnels}}, $self->fork_websocket_data_tunnel($info->{nbd}, $info->{nbd}, 1);
+	}
+    } else {
+	$self->log('info', "start remote tunnel");
+
+	if ($migration_type eq 'secure') {
+	    if ($info->{ruri} =~ /^unix:/) {
+		unlink $info->{raddr};
+		$self->{tunnel} = $self->fork_tunnel("$info->{raddr}:$info->{raddr}");
+		$self->{tunnel}->{sock_addr} = $info->{raddr};
+
+		my $unix_socket_try = 0; # wait for the socket to become ready
+		while (! -S $info->{raddr}) {
+		    $unix_socket_try++;
+		    if ($unix_socket_try > 100) {
+			$self->{errors} = 1;
+			$self->finish_tunnel($self->{tunnel});
+			die "Timeout, migration socket $info->{ruri} did not get ready";
+		    }
 
-	if ($ruri =~ /^unix:/) {
-	    unlink $raddr;
-	    $self->{tunnel} = $self->fork_tunnel("$raddr:$raddr");
-	    $self->{tunnel}->{sock_addr} = $raddr;
+		    usleep(50000);
+		}
 
-	    my $unix_socket_try = 0; # wait for the socket to become ready
-	    while (! -S $raddr) {
-		$unix_socket_try++;
-		if ($unix_socket_try > 100) {
-		    $self->{errors} = 1;
-		    $self->finish_tunnel($self->{tunnel});
-		    die "Timeout, migration socket $ruri did not get ready";
+	    } elsif ($info->{ruri} =~ /^tcp:/) {
+		my $tunnel_addr;
+		if ($info->{raddr} eq "localhost") {
+		    # for backwards compatibility with older qemu-server versions
+		    my $pfamily = PVE::Tools::get_host_address_family($nodename);
+		    my $lport = PVE::Tools::next_migrate_port($pfamily);
+		    $tunnel_addr = "$lport:localhost:$info->{rport}";
 		}
 
-		usleep(50000);
-	    }
+		$self->{tunnel} = $self->fork_tunnel($tunnel_addr);
 
-	} elsif ($ruri =~ /^tcp:/) {
-	    my $tunnel_addr;
-	    if ($raddr eq "localhost") {
-		# for backwards compatibility with older qemu-server versions
-		my $pfamily = PVE::Tools::get_host_address_family($nodename);
-		my $lport = PVE::Tools::next_migrate_port($pfamily);
-		$tunnel_addr = "$lport:localhost:$rport";
+	    } else {
+		die "unsupported protocol in migration URI: $info->{ruri}\n";
 	    }
-
-	    $self->{tunnel} = $self->fork_tunnel($tunnel_addr);
-
 	} else {
-	    die "unsupported protocol in migration URI: $ruri\n";
+	    #fork tunnel for insecure migration, to send faster commands like resume
+	    $self->{tunnel} = $self->fork_tunnel();
 	}
-    } else {
-	#fork tunnel for insecure migration, to send faster commands like resume
-	$self->{tunnel} = $self->fork_tunnel();
     }
 
     my $start = time();
@@ -688,10 +883,13 @@ sub phase2 {
 	    my $nbd_uri = $target->{nbd_uri};
 
 	    my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive});
-	    my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr});
-
 	    my $source_sid = PVE::Storage::Plugin::parse_volume_id($source_drive->{file});
-	    my $target_sid = PVE::Storage::Plugin::parse_volume_id($target_drive->{file});
+
+	    my $target_sid;
+	    if (!$self->{opts}->{remote}) {
+		my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr});
+		$target_sid = PVE::Storage::Plugin::parse_volume_id($target_drive->{file});
+	    }
 
 	    my $bwlimit = PVE::Storage::get_bandwidth_limit('migrate', [$source_sid, $target_sid], $opt_bwlimit);
 
@@ -700,7 +898,7 @@ sub phase2 {
 	}
     }
 
-    $self->log('info', "starting online/live migration on $ruri");
+    $self->log('info', "starting online/live migration on $info->{ruri}");
     $self->{livemigration} = 1;
 
     # load_defaults
@@ -755,7 +953,7 @@ sub phase2 {
     };
     $self->log('info', "migrate-set-parameters error: $@") if $@;
 
-    if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
+    if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) {
 	my $rpcenv = PVE::RPCEnvironment::get();
 	my $authuser = $rpcenv->get_user();
 
@@ -768,19 +966,19 @@ sub phase2 {
 
 	eval {
 	    mon_cmd($vmid, "client_migrate_info", protocol => 'spice',
-						hostname => $proxyticket, 'port' => 0, 'tls-port' => $spice_port,
+						hostname => $proxyticket, 'port' => 0, 'tls-port' => $info->{spice_port},
 						'cert-subject' => $subject);
 	};
 	$self->log('info', "client_migrate_info error: $@") if $@;
 
     }
 
-    $self->log('info', "start migrate command to $ruri");
+    $self->log('info', "start migrate command to $info->{ruri}");
     eval {
-	mon_cmd($vmid, "migrate", uri => $ruri);
+	mon_cmd($vmid, "migrate", uri => $info->{ruri});
     };
     my $merr = $@;
-    $self->log('info', "migrate uri => $ruri failed: $merr") if $merr;
+    $self->log('info', "migrate uri => $info->{ruri} failed: $merr") if $merr;
 
     my $lstat = 0;
     my $usleep = 1000000;
@@ -918,11 +1116,15 @@ sub phase2_cleanup {
 
     my $nodename = PVE::INotify::nodename();
 
-    my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
-    eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
-    if (my $err = $@) {
-        $self->log('err', $err);
-        $self->{errors} = 1;
+    if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) {
+	$self->write_tunnel($self->{tunnel}, 10, "stop");
+    } else {
+	my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
+	eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+	if (my $err = $@) {
+            $self->log('err', $err);
+            $self->{errors} = 1;
+	}
     }
 
     if ($self->{tunnel}) {
@@ -941,6 +1143,10 @@ sub phase3 {
     return if $self->{phase2errors};
 
     # destroy local copies
+
+    # FIXME remove early return for real migration!
+    return;
+
     foreach my $volid (@$volids) {
 	eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
 	if (my $err = $@) {
@@ -968,35 +1174,46 @@ sub phase3_cleanup {
 	    eval { PVE::QemuMigrate::cleanup_remotedisks($self) };
 	    die "Failed to complete storage migration: $err\n";
 	} else {
+	    # TODO handle properly
+	    if (!$self->{opts}->{remote}) {
 	    foreach my $target_drive (keys %{$self->{target_drive}}) {
 		my $drive = PVE::QemuServer::parse_drive($target_drive, $self->{target_drive}->{$target_drive}->{drivestr});
 		$conf->{$target_drive} = PVE::QemuServer::print_drive($drive);
 		PVE::QemuConfig->write_config($vmid, $conf);
 	    }
+	    }
 	}
     }
 
     # transfer replication state before move config
     $self->transfer_replication_state() if $self->{replicated_volumes};
 
-    # move config to remote node
-    my $conffile = PVE::QemuConfig->config_file($vmid);
-    my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node});
+    if ($self->{opts}->{remote}) {
+	# TODO delete local config?
+    } else {
+	# move config to remote node
+	my $conffile = PVE::QemuConfig->config_file($vmid);
+	my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node});
 
-    die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
-        if !rename($conffile, $newconffile);
+	die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
+	    if !rename($conffile, $newconffile);
+    }
 
     $self->switch_replication_job_target() if $self->{replicated_volumes};
 
     if ($self->{livemigration}) {
 	if ($self->{storage_migration}) {
-	    # stop nbd server on remote vm - requirement for resume since 2.9
-	    my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
+	    if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) {
+		$self->write_tunnel($tunnel, 30, 'nbdstop');
+	    } else {
+	    	# stop nbd server on remote vm - requirement for resume since 2.9
+	    	my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
 
-	    eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
-	    if (my $err = $@) {
-		$self->log('err', $err);
-		$self->{errors} = 1;
+	    	eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+	    	if (my $err = $@) {
+		    $self->log('err', $err);
+		    $self->{errors} = 1;
+	    	}
 	    }
 	}
 
@@ -1030,16 +1247,30 @@ sub phase3_cleanup {
 
     # close tunnel on successful migration, on error phase2_cleanup closed it
     if ($tunnel) {
-	eval { finish_tunnel($self, $tunnel);  };
-	if (my $err = $@) {
-	    $self->log('err', $err);
-	    $self->{errors} = 1;
+	if ($tunnel->{version} == 1) {
+	    eval { finish_tunnel($self, $tunnel);  };
+	    if (my $err = $@) {
+		$self->log('err', $err);
+		$self->{errors} = 1;
+	    }
+	    $tunnel = undef;
+	    delete $self->{tunnel};
+	} else {
+	    foreach my $data_tunnel (@{$self->{data_tunnels}}) {
+		eval {
+		    kill(15, $data_tunnel->{pid});
+		};
+		if (my $err = $@) {
+		    $self->log('err', $err);
+		    $self->{errors} = 1;
+		}
+	    }
 	}
     }
 
     eval {
 	my $timer = 0;
-	if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) {
+	if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running} && !$self->{opts}->{remote}) {
 	    $self->log('info', "Waiting for spice server migration");
 	    while (1) {
 		my $res = mon_cmd($vmid, 'query-spice');
@@ -1072,6 +1303,9 @@ sub phase3_cleanup {
 	# destroy local copies
 	my $volids = $self->{online_local_volumes};
 
+	# TODO remove for proper migration!
+	if (!$self->{opts}->{remote}) {
+
 	foreach my $volid (@$volids) {
 	    eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
 	    if (my $err = $@) {
@@ -1080,12 +1314,20 @@ sub phase3_cleanup {
 		last if $err =~ /^interrupted by signal$/;
 	    }
 	}
+	}
 
     }
 
+    # TODO remove local config for proper remote migration
+
     # clear migrate lock
-    my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
-    $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
+    if ($tunnel) {
+	$self->write_tunnel($tunnel, 10, "unlock");
+	$self->finish_tunnel($tunnel);
+    } else {
+	my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
+	$self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
+    }
 }
 
 sub final_cleanup {
-- 
2.20.1





More information about the pve-devel mailing list