[pve-devel] [PATCH qemu-server 4/4] implement PoC migration to remote cluster/node
Fabian Grünbichler
f.gruenbichler at proxmox.com
Fri Mar 6 11:20:35 CET 2020
there's obviously lots of TODOs and FIXMEs in here, the big ones are:
- better handling of storage switching
- handling of network switching
- implementing cleanup
- actually dropping the local/source config and disks
- NBD client side is kept open by Qemu, so we need to terminate the data
tunnel in order to stop the VM
I think it would make sense to re-factor vm_start and pass in all the
(mostly) migration-related parameters as a hash. that way, we could tell
the remote side that this is a remote incoming migration, and split out
the disk/network updating into its own step in mtunnel that can happen
before, and then start NBD for all disks, not just local, newly
allocated ones.
for mapping disks/NICs I see three options:
- global switch to map everything to one target, with fall-back to keep
as-is (current)
- mapping of source storage/bridge to target storage/bridge
- mapping of source disks (volid)/NIC to target storage/bridge
the intention of sending this in the current state is mainly to get
feedback early on on whether we want to go down this route, or use a
totally different approach.
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
PVE/QemuMigrate.pm | 510 +++++++++++++++++++++++++++++++++------------
1 file changed, 376 insertions(+), 134 deletions(-)
diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm
index 6db5e62..01a5b12 100644
--- a/PVE/QemuMigrate.pm
+++ b/PVE/QemuMigrate.pm
@@ -3,9 +3,18 @@ package PVE::QemuMigrate;
use strict;
use warnings;
use PVE::AbstractMigrate;
+
+use Data::Dumper;
+
use IO::File;
+use IO::Socket::UNIX;
use IPC::Open2;
+use JSON;
+use MIME::Base64;
use POSIX qw( WNOHANG );
+use URI::Escape;
+
+use PVE::APIClient::LWP;
use PVE::INotify;
use PVE::Tools;
use PVE::Cluster;
@@ -15,9 +24,11 @@ use PVE::QemuServer::Machine;
use PVE::QemuServer::Monitor qw(mon_cmd);
use Time::HiRes qw( usleep );
use PVE::RPCEnvironment;
+use PVE::RemoteConfig;
use PVE::ReplicationConfig;
use PVE::ReplicationState;
use PVE::Replication;
+use PVE::WebSocket;
use base qw(PVE::AbstractMigrate);
@@ -56,8 +67,8 @@ sub finish_command_pipe {
my $writer = $cmdpipe->{writer};
my $reader = $cmdpipe->{reader};
- $writer->close();
- $reader->close();
+ $writer->close() if defined($writer);
+ $reader->close() if defined($reader);
my $collect_child_process = sub {
my $res = waitpid($cpid, WNOHANG);
@@ -173,6 +184,109 @@ sub fork_tunnel {
return $tunnel;
}
+sub fork_websocket_tunnel {
+ my ($self) = @_;
+
+ my $remote = $self->{opts}->{remote};
+ my $remote_config = PVE::RemoteConfig->new();
+ my ($ips, $conn_args) = $remote_config->get_remote_info($self->{opts}->{remote}, $self->{node});
+ my $conn = PVE::APIClient::LWP->new(%$conn_args);
+
+ my $mtunnel_worker = $conn->post("api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnel", { version => 2 });
+ $self->log('info', "Spawned migration tunnel worker on '$self->{opts}->{remote}/$self->{node}'");
+ # FIXME add mtunnel ticket
+ my $api_path = "/api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnelwebsocket?socket=".uri_escape("/run/qemu-server/$self->{vmid}.mtunnel");
+
+ # TODO use migration network?
+ my $ws = PVE::WebSocket->new($conn->{host}, 8006, $api_path);
+
+ my $auth_header = "Authorization: $conn->{apitoken}";
+
+ $ws->connect($auth_header);
+
+ my $reader = IO::Pipe->new();
+ my $writer = IO::Pipe->new();
+
+ my $cpid = fork();
+ if ($cpid) {
+ my $tunnel = { writer => $writer->writer(), reader => $reader->reader(), socket => $ws, pid => $cpid, version => 2 };
+ # FIXME: read version
+ print "hello: '".$self->read_tunnel($tunnel, 10)."'\n";
+ print "hello: '".$self->read_tunnel($tunnel, 10)."'\n";
+ $self->{api_conn} = $conn;
+ $self->{data_tunnels} = [];
+ return $tunnel;
+ } else {
+ $ws->{reader} = $writer->reader();
+ $ws->{writer} = $reader->writer();
+ $ws->process();
+ exit 0;
+ }
+}
+
+sub fork_websocket_data_tunnel {
+ my ($self, $local, $remote) = @_;
+ # TODO implement TCP?
+
+ unlink $local;
+ my $local_socket = IO::Socket::UNIX->new(
+ Type => SOCK_STREAM(),
+ Local => $local,
+ Listen => 5,
+ );
+
+ die "could not bind to local socket '$local' of data tunnel - $!\n"
+ if !$local_socket;
+
+ my $cpid = fork();
+ if ($cpid) {
+ $local_socket->close();
+ return { local => $local, pid => $cpid };
+ } else {
+ $self->log('info', "forked websocket data tunnel $local!");
+ my $clients = [];
+ my $exit_handler = sub {
+ $self->log('info', "Closing data tunnel '$local'");
+ $local_socket->close();
+ foreach my $c (@$clients) {
+ $self->log('info', "closing client socket for $local #$c->{index}");
+ $self->finish_command_pipe($c);
+ }
+ $self->log('info', "Closed data tunnel '$local'");
+ };
+ local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = $exit_handler;
+ my $index = 0;
+ while (my $client = $local_socket->accept()) {
+ $index++;
+ $self->log('info', "accepted new connection #$index on $local, establishing web socket connection..\n");
+ $cpid = fork();
+ if ($cpid) {
+ $client->close();
+ push @$clients, { pid => $cpid, index => $index };
+ } else {
+ $local_socket->close();
+ local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = 'IGNORE';
+ my $conn = $self->{api_conn};
+ my $api_path = "/api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnelwebsocket?socket=".uri_escape($remote);
+ my $ws = PVE::WebSocket->new($conn->{host}, 8006, $api_path);
+
+ my $auth_header = "Authorization: $conn->{apitoken}";
+
+ $ws->connect($auth_header);
+ local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub { $ws->close() };
+
+ $self->log('info', "connected websocket data tunnel $local #$index");
+ $ws->{reader} = $client;
+ $ws->{writer} = $client;
+ $ws->process();
+ $self->log('info', "processing finished on websocket data tunnel $local #$index..");
+ exit 0;
+ }
+ }
+ exit 0;
+ }
+}
+
sub finish_tunnel {
my ($self, $tunnel) = @_;
@@ -239,6 +353,12 @@ sub prepare {
my $targetsid = $self->{opts}->{targetstorage} // $sid;
my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
+
+ if ($self->{opts}->{remote}) {
+ push @$need_activate, $volid;
+ next;
+ }
+
PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
if ($scfg->{shared}) {
@@ -256,10 +376,16 @@ sub prepare {
# activate volumes
PVE::Storage::activate_volumes($self->{storecfg}, $need_activate);
- # test ssh connection
- my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
- eval { $self->cmd_quiet($cmd); };
- die "Can't connect to destination address using public key\n" if $@;
+ if ($self->{opts}->{remote}) {
+ # test web socket connection
+ $self->{tunnel} = $self->fork_websocket_tunnel();
+ print "websocket tunnel started\n";
+ } else {
+ # test ssh connection
+ my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
+ eval { $self->cmd_quiet($cmd); };
+ die "Can't connect to destination address using public key\n" if $@;
+ }
return $running;
}
@@ -296,7 +422,7 @@ sub sync_disks {
my @sids = PVE::Storage::storage_ids($self->{storecfg});
foreach my $storeid (@sids) {
my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
- next if $scfg->{shared};
+ next if $scfg->{shared} && !$self->{opts}->{remote};
next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1);
# get list from PVE::Storage (for unused volumes)
@@ -307,7 +433,8 @@ sub sync_disks {
my $targetsid = $override_targetsid // $storeid;
# check if storage is available on target node
- PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
+ PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node})
+ if !$self->{opts}->{remote};
PVE::Storage::foreach_volid($dl, sub {
my ($volid, $sid, $volname) = @_;
@@ -345,9 +472,10 @@ sub sync_disks {
my $targetsid = $override_targetsid // $sid;
# check if storage is available on both nodes
my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid);
- PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node});
+ PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node})
+ if !$self->{opts}->{remote};
- return if $scfg->{shared};
+ return if $scfg->{shared} && !$self->{opts}->{remote};
$local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot';
@@ -423,6 +551,9 @@ sub sync_disks {
my $migratable = $scfg->{type} =~ /^(?:dir|zfspool|lvmthin|lvm)$/;
+ # TODO: implement properly
+ $migratable = 1 if $self->{opts}->{remote};
+
die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n"
if !$migratable;
@@ -435,6 +566,9 @@ sub sync_disks {
my $rep_cfg = PVE::ReplicationConfig->new();
if (my $jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node})) {
die "can't live migrate VM with replicated volumes\n" if $self->{running};
+ die "can't migrate VM with replicated volumes to remote cluster/node\n"
+ if $self->{opts}->{remote};
+
$self->log('info', "replicating disk images");
my $start_time = time();
my $logfunc = sub { $self->log('info', shift) };
@@ -469,6 +603,9 @@ sub sync_disks {
next;
} else {
next if $self->{replicated_volumes}->{$volid};
+ die "storage migration to remote cluster/node not implemented yet\n"
+ if $self->{opts}->{remote};
+
push @{$self->{volumes}}, $volid;
my $opts = $self->{opts};
my $insecure = $opts->{migration_type} eq 'insecure';
@@ -489,8 +626,12 @@ sub sync_disks {
sub cleanup_remotedisks {
my ($self) = @_;
- foreach my $target_drive (keys %{$self->{target_drive}}) {
+ if ($self->{opts}->{remote}) {
+ warn "cleanup not yet implemented";
+ return;
+ }
+ foreach my $target_drive (keys %{$self->{target_drive}}) {
my $drive = PVE::QemuServer::parse_drive($target_drive, $self->{target_drive}->{$target_drive}->{drivestr});
my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
@@ -520,6 +661,32 @@ sub phase1 {
# sync_disks fixes disk sizes to match their actual size, write changes so
# target allocates correct volumes
PVE::QemuConfig->write_config($vmid, $conf);
+
+ if ($self->{opts}->{remote}) {
+ # TODO without disks here, send disks + nics as separate commands?
+ # we want single line here!
+ my $remote_conf = PVE::QemuConfig->load_config($vmid);
+ if (my $targetsid = $self->{opts}->{targetstorage}) {
+ PVE::QemuServer::foreach_drive($remote_conf, sub {
+ my ($ds, $drive) = @_;
+
+ return if PVE::QemuServer::drive_is_cdrom($drive);
+
+ my $volid = $drive->{file};
+ return if !$volid;
+ return if ! grep { $_ eq $volid} @{$self->{online_local_volumes}};
+
+ $self->log('info', "Rewriting config to move '$ds' - '$volid' to '$targetsid'");
+
+ my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
+ $drive->{file} = "$targetsid:$volname";
+ $remote_conf->{$ds} = PVE::QemuServer::print_drive($drive);
+ });
+ }
+ my $conf_str = MIME::Base64::encode_base64url(JSON::encode_json($remote_conf));
+ $self->write_tunnel($self->{tunnel}, 10, "config $conf_str");
+ }
+
};
sub phase1_cleanup {
@@ -549,13 +716,9 @@ sub phase2 {
$self->log('info', "starting VM $vmid on remote node '$self->{node}'");
- my $raddr;
- my $rport;
- my $ruri; # the whole migration dst. URI (protocol:address[:port])
- my $nodename = PVE::INotify::nodename();
+ my $info = {};
- ## start on remote node
- my $cmd = [@{$self->{rem_ssh}}];
+ my $nodename = PVE::INotify::nodename();
my $spice_ticket;
if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
@@ -563,113 +726,145 @@ sub phase2 {
$spice_ticket = $res->{ticket};
}
- push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename;
-
my $migration_type = $self->{opts}->{migration_type};
+ my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix';
- push @$cmd, '--migration_type', $migration_type;
-
- push @$cmd, '--migration_network', $self->{opts}->{migration_network}
- if $self->{opts}->{migration_network};
-
- if ($migration_type eq 'insecure') {
- push @$cmd, '--stateuri', 'tcp';
+ ## start on remote node
+ if ($self->{opts}->{remote}) {
+ die "insecure migration to remote cluster/node not implemented yet\n"
+ if $migration_type eq 'insecure';
+
+ my $start_json = JSON::encode_json({
+ spice => $spice_ticket,
+ migration_type => $migration_type,
+ state_uri => $state_uri,
+ network => $self->{opts}->{migration_network},
+ machine => $self->{forcemachine},
+ targetstorage => $self->{opts}->{targetstorage},
+ });
+ $self->write_tunnel($self->{tunnel}, 10, "start $start_json");
+ $info = JSON::decode_json($self->read_tunnel($self->{tunnel}, 10));
+ print Dumper($info), "\n";
} else {
- push @$cmd, '--stateuri', 'unix';
- }
+ my $cmd = [@{$self->{rem_ssh}}];
+ push @$cmd , 'qm', 'start', $vmid;
+ push @$cmd, '--skiplock', '--migratedfrom', $nodename;
+ push @$cmd, '--migration_type', $migration_type;
+ push @$cmd, '--migration_network', $self->{opts}->{migration_network}
+ if $self->{opts}->{migration_network};
- if ($self->{forcemachine}) {
- push @$cmd, '--machine', $self->{forcemachine};
- }
+ push @$cmd, '--stateuri', $state_uri;
- if ($self->{online_local_volumes}) {
- push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
- }
+ if ($self->{forcemachine}) {
+ push @$cmd, '--machine', $self->{forcemachine};
+ }
- my $spice_port;
+ if ($self->{online_local_volumes}) {
+ push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
+ }
- # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
- # instead we pipe it through STDIN
- my $exitcode = PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
- my $line = shift;
+ # Note: We try to keep $spice_ticket secret (do not pass via command line parameter)
+ # instead we pipe it through STDIN
+ my $exitcode = PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub {
+ my $line = shift;
- if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
- $raddr = $1;
- $rport = int($2);
- $ruri = "tcp:$raddr:$rport";
- }
- elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
- $raddr = $1;
- die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
- $ruri = "unix:$raddr";
- }
- elsif ($line =~ m/^migration listens on port (\d+)$/) {
- $raddr = "localhost";
- $rport = int($1);
- $ruri = "tcp:$raddr:$rport";
- }
- elsif ($line =~ m/^spice listens on port (\d+)$/) {
- $spice_port = int($1);
- }
- elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
- my $drivestr = $4;
- my $nbd_uri = "nbd:$1:$2:exportname=$3";
- my $targetdrive = $3;
- $targetdrive =~ s/drive-//g;
+ if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
+ $info->{raddr} = $1;
+ $info->{rport} = int($2);
+ $info->{ruri} = "tcp:$info->{raddr}:$info->{rport}";
+ }
+ elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
+ $info->{raddr} = $1;
+ die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
+ $info->{ruri} = "unix:$info->{raddr}";
+ }
+ elsif ($line =~ m/^migration listens on port (\d+)$/) {
+ $info->{raddr} = "localhost";
+ $info->{rport} = int($1);
+ $info->{ruri} = "tcp:$info->{raddr}:$info->{rport}";
+ }
+ elsif ($line =~ m/^spice listens on port (\d+)$/) {
+ $info->{spice_port} = int($1);
+ }
+ elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) {
+ my $drivestr = $4;
+ my $nbd_uri = "nbd:$1:$2:exportname=$3";
+ my $targetdrive = $3;
+ $targetdrive =~ s/drive-//g;
- $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr;
- $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri;
+ $info->{drives}->{$targetdrive}->{drivestr} = $drivestr;
+ $info->{drives}->{$targetdrive}->{nbd_uri} = $nbd_uri;
- } elsif ($line =~ m/^QEMU: (.*)$/) {
- $self->log('info', "[$self->{node}] $1\n");
- }
- }, errfunc => sub {
- my $line = shift;
- $self->log('info', "[$self->{node}] $line");
- }, noerr => 1);
+ } elsif ($line =~ m/^QEMU: (.*)$/) {
+ $self->log('info', "[$self->{node}] $1\n");
+ }
+ }, errfunc => sub {
+ my $line = shift;
+ $self->log('info', "[$self->{node}] $line");
+ }, noerr => 1);
- die "remote command failed with exit code $exitcode\n" if $exitcode;
+ die "remote command failed with exit code $exitcode\n" if $exitcode;
+ }
- die "unable to detect remote migration address\n" if !$raddr;
+ die "unable to detect remote migration address\n" if !$info->{raddr};
- $self->log('info', "start remote tunnel");
+ foreach my $drive (keys %{$info->{drives}}) {
+ $self->{target_drive}->{$drive}->{drivestr} = $info->{drives}->{$drive}->{drivestr};
+ $self->{target_drive}->{$drive}->{nbd_uri} = $info->{drives}->{$drive}->{nbd_uri};
+ }
- if ($migration_type eq 'secure') {
+ if ($self->{tunnel} && $self->{opts}->{remote}) {
+ # TODO support TCP?
+ if ($info->{ruri} =~ /^unix:/) {
+ $self->log('info', "starting memory migration tunnel '$info->{raddr}' => '$info->{raddr}'");
+ push @{$self->{data_tunnels}}, $self->fork_websocket_data_tunnel($info->{raddr}, $info->{raddr});
+ } else {
+ die "unsupported migration uri '$info->{ruri}'\n";
+ }
+ if ($info->{nbd}) {
+ $self->log('info', "starting local disk migration tunnel '$info->{nbd}' => '$info->{nbd}'");
+ # keep open - we might have multiple disks!
+ push @{$self->{data_tunnels}}, $self->fork_websocket_data_tunnel($info->{nbd}, $info->{nbd}, 1);
+ }
+ } else {
+ $self->log('info', "start remote tunnel");
+
+ if ($migration_type eq 'secure') {
+ if ($info->{ruri} =~ /^unix:/) {
+ unlink $info->{raddr};
+ $self->{tunnel} = $self->fork_tunnel("$info->{raddr}:$info->{raddr}");
+ $self->{tunnel}->{sock_addr} = $info->{raddr};
+
+ my $unix_socket_try = 0; # wait for the socket to become ready
+ while (! -S $info->{raddr}) {
+ $unix_socket_try++;
+ if ($unix_socket_try > 100) {
+ $self->{errors} = 1;
+ $self->finish_tunnel($self->{tunnel});
+ die "Timeout, migration socket $info->{ruri} did not get ready";
+ }
- if ($ruri =~ /^unix:/) {
- unlink $raddr;
- $self->{tunnel} = $self->fork_tunnel("$raddr:$raddr");
- $self->{tunnel}->{sock_addr} = $raddr;
+ usleep(50000);
+ }
- my $unix_socket_try = 0; # wait for the socket to become ready
- while (! -S $raddr) {
- $unix_socket_try++;
- if ($unix_socket_try > 100) {
- $self->{errors} = 1;
- $self->finish_tunnel($self->{tunnel});
- die "Timeout, migration socket $ruri did not get ready";
+ } elsif ($info->{ruri} =~ /^tcp:/) {
+ my $tunnel_addr;
+ if ($info->{raddr} eq "localhost") {
+ # for backwards compatibility with older qemu-server versions
+ my $pfamily = PVE::Tools::get_host_address_family($nodename);
+ my $lport = PVE::Tools::next_migrate_port($pfamily);
+ $tunnel_addr = "$lport:localhost:$info->{rport}";
}
- usleep(50000);
- }
+ $self->{tunnel} = $self->fork_tunnel($tunnel_addr);
- } elsif ($ruri =~ /^tcp:/) {
- my $tunnel_addr;
- if ($raddr eq "localhost") {
- # for backwards compatibility with older qemu-server versions
- my $pfamily = PVE::Tools::get_host_address_family($nodename);
- my $lport = PVE::Tools::next_migrate_port($pfamily);
- $tunnel_addr = "$lport:localhost:$rport";
+ } else {
+ die "unsupported protocol in migration URI: $info->{ruri}\n";
}
-
- $self->{tunnel} = $self->fork_tunnel($tunnel_addr);
-
} else {
- die "unsupported protocol in migration URI: $ruri\n";
+ #fork tunnel for insecure migration, to send faster commands like resume
+ $self->{tunnel} = $self->fork_tunnel();
}
- } else {
- #fork tunnel for insecure migration, to send faster commands like resume
- $self->{tunnel} = $self->fork_tunnel();
}
my $start = time();
@@ -688,10 +883,13 @@ sub phase2 {
my $nbd_uri = $target->{nbd_uri};
my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive});
- my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr});
-
my $source_sid = PVE::Storage::Plugin::parse_volume_id($source_drive->{file});
- my $target_sid = PVE::Storage::Plugin::parse_volume_id($target_drive->{file});
+
+ my $target_sid;
+ if (!$self->{opts}->{remote}) {
+ my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr});
+ $target_sid = PVE::Storage::Plugin::parse_volume_id($target_drive->{file});
+ }
my $bwlimit = PVE::Storage::get_bandwidth_limit('migrate', [$source_sid, $target_sid], $opt_bwlimit);
@@ -700,7 +898,7 @@ sub phase2 {
}
}
- $self->log('info', "starting online/live migration on $ruri");
+ $self->log('info', "starting online/live migration on $info->{ruri}");
$self->{livemigration} = 1;
# load_defaults
@@ -755,7 +953,7 @@ sub phase2 {
};
$self->log('info', "migrate-set-parameters error: $@") if $@;
- if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
+ if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) {
my $rpcenv = PVE::RPCEnvironment::get();
my $authuser = $rpcenv->get_user();
@@ -768,19 +966,19 @@ sub phase2 {
eval {
mon_cmd($vmid, "client_migrate_info", protocol => 'spice',
- hostname => $proxyticket, 'port' => 0, 'tls-port' => $spice_port,
+ hostname => $proxyticket, 'port' => 0, 'tls-port' => $info->{spice_port},
'cert-subject' => $subject);
};
$self->log('info', "client_migrate_info error: $@") if $@;
}
- $self->log('info', "start migrate command to $ruri");
+ $self->log('info', "start migrate command to $info->{ruri}");
eval {
- mon_cmd($vmid, "migrate", uri => $ruri);
+ mon_cmd($vmid, "migrate", uri => $info->{ruri});
};
my $merr = $@;
- $self->log('info', "migrate uri => $ruri failed: $merr") if $merr;
+ $self->log('info', "migrate uri => $info->{ruri} failed: $merr") if $merr;
my $lstat = 0;
my $usleep = 1000000;
@@ -918,11 +1116,15 @@ sub phase2_cleanup {
my $nodename = PVE::INotify::nodename();
- my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
- eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
+ if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) {
+ $self->write_tunnel($self->{tunnel}, 10, "stop");
+ } else {
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
+ eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
}
if ($self->{tunnel}) {
@@ -941,6 +1143,10 @@ sub phase3 {
return if $self->{phase2errors};
# destroy local copies
+
+ # FIXME remove early return for real migration!
+ return;
+
foreach my $volid (@$volids) {
eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
if (my $err = $@) {
@@ -968,35 +1174,46 @@ sub phase3_cleanup {
eval { PVE::QemuMigrate::cleanup_remotedisks($self) };
die "Failed to complete storage migration: $err\n";
} else {
+ # TODO handle properly
+ if (!$self->{opts}->{remote}) {
foreach my $target_drive (keys %{$self->{target_drive}}) {
my $drive = PVE::QemuServer::parse_drive($target_drive, $self->{target_drive}->{$target_drive}->{drivestr});
$conf->{$target_drive} = PVE::QemuServer::print_drive($drive);
PVE::QemuConfig->write_config($vmid, $conf);
}
+ }
}
}
# transfer replication state before move config
$self->transfer_replication_state() if $self->{replicated_volumes};
- # move config to remote node
- my $conffile = PVE::QemuConfig->config_file($vmid);
- my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node});
+ if ($self->{opts}->{remote}) {
+ # TODO delete local config?
+ } else {
+ # move config to remote node
+ my $conffile = PVE::QemuConfig->config_file($vmid);
+ my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node});
- die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
- if !rename($conffile, $newconffile);
+ die "Failed to move config to node '$self->{node}' - rename failed: $!\n"
+ if !rename($conffile, $newconffile);
+ }
$self->switch_replication_job_target() if $self->{replicated_volumes};
if ($self->{livemigration}) {
if ($self->{storage_migration}) {
- # stop nbd server on remote vm - requirement for resume since 2.9
- my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
+ if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) {
+ $self->write_tunnel($tunnel, 30, 'nbdstop');
+ } else {
+ # stop nbd server on remote vm - requirement for resume since 2.9
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
- eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
+ eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
}
}
@@ -1030,16 +1247,30 @@ sub phase3_cleanup {
# close tunnel on successful migration, on error phase2_cleanup closed it
if ($tunnel) {
- eval { finish_tunnel($self, $tunnel); };
- if (my $err = $@) {
- $self->log('err', $err);
- $self->{errors} = 1;
+ if ($tunnel->{version} == 1) {
+ eval { finish_tunnel($self, $tunnel); };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
+ $tunnel = undef;
+ delete $self->{tunnel};
+ } else {
+ foreach my $data_tunnel (@{$self->{data_tunnels}}) {
+ eval {
+ kill(15, $data_tunnel->{pid});
+ };
+ if (my $err = $@) {
+ $self->log('err', $err);
+ $self->{errors} = 1;
+ }
+ }
}
}
eval {
my $timer = 0;
- if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) {
+ if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running} && !$self->{opts}->{remote}) {
$self->log('info', "Waiting for spice server migration");
while (1) {
my $res = mon_cmd($vmid, 'query-spice');
@@ -1072,6 +1303,9 @@ sub phase3_cleanup {
# destroy local copies
my $volids = $self->{online_local_volumes};
+ # TODO remove for proper migration!
+ if (!$self->{opts}->{remote}) {
+
foreach my $volid (@$volids) {
eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
if (my $err = $@) {
@@ -1080,12 +1314,20 @@ sub phase3_cleanup {
last if $err =~ /^interrupted by signal$/;
}
}
+ }
}
+ # TODO remove local config for proper remote migration
+
# clear migrate lock
- my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
- $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
+ if ($tunnel) {
+ $self->write_tunnel($tunnel, 10, "unlock");
+ $self->finish_tunnel($tunnel);
+ } else {
+ my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
+ $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
+ }
}
sub final_cleanup {
--
2.20.1
More information about the pve-devel
mailing list