[pve-devel] [PATCH v3 qemu-server 09/10] migrate: add remote migration handling
Fabian Ebner
f.ebner at proxmox.com
Tue Jan 4 14:58:03 CET 2022
Two comments inline:
Am 22.12.21 um 14:52 schrieb Fabian Grünbichler:
> remote migration uses a websocket connection to a task worker running on
> the target node instead of commands via SSH to control the migration.
> this websocket tunnel is started earlier than the SSH tunnel, and allows
> adding UNIX-socket forwarding over additional websocket connections
> on-demand.
>
> the main differences to regular intra-cluster migration are:
> - source VM config and disks are only removed upon request via --delete
> - shared storages are treated like local storages, since we can't
> assume they are shared across clusters (with potentical to extend this
> by marking storages as shared)
> - NBD migrated disks are explicitly pre-allocated on the target node via
> tunnel command before starting the target VM instance
> - in addition to storages, network bridges and the VMID itself is
> transformed via a user defined mapping
> - all commands and migration data streams are sent via a WS tunnel proxy
>
> Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
> ---
>
> Notes:
> requires bumped pve-guest-common
>
> v3:
> - move WS tunnel helpers to pve-guest-common-perl
> - check bridge mapping early
>
> v2:
> - improve tunnel version info printing and error handling
> - don't cleanup unix sockets twice
> - url escape remote socket path
> - cleanup nits and small issues
>
> requires proxmox-websocket-tunnel
>
> v3:
> - fix misplaced parentheses
>
> v2:
> - improve tunnel version info printing and error handling
> - don't cleanup unix sockets twice
> - url escape remote socket path
> - cleanup nits and small issues
>
> PVE/API2/Qemu.pm | 2 +-
> PVE/QemuMigrate.pm | 425 +++++++++++++++++++++++++++++++++++++--------
> PVE/QemuServer.pm | 7 +-
> 3 files changed, 359 insertions(+), 75 deletions(-)
>
> diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
> index d188b77..cf90fe7 100644
> --- a/PVE/API2/Qemu.pm
> +++ b/PVE/API2/Qemu.pm
> @@ -4803,7 +4803,7 @@ __PACKAGE__->register_method({
> # bump/reset for breaking changes
> # bump/bump for opt-in changes
> return {
> - api => 2,
> + api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
> age => 0,
> };
> },
> diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm
> index 897018b..3e9d29e 100644
> --- a/PVE/QemuMigrate.pm
> +++ b/PVE/QemuMigrate.pm
> @@ -5,11 +5,10 @@ use warnings;
>
> use IO::File;
> use IPC::Open2;
> -use POSIX qw( WNOHANG );
> use Time::HiRes qw( usleep );
>
> -use PVE::Format qw(render_bytes);
> use PVE::Cluster;
> +use PVE::Format qw(render_bytes);
> use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne);
> use PVE::INotify;
> use PVE::RPCEnvironment;
> @@ -17,6 +16,7 @@ use PVE::Replication;
> use PVE::ReplicationConfig;
> use PVE::ReplicationState;
> use PVE::Storage;
> +use PVE::StorageTunnel;
> use PVE::Tools;
> use PVE::Tunnel;
>
> @@ -31,6 +31,9 @@ use PVE::QemuServer;
> use PVE::AbstractMigrate;
> use base qw(PVE::AbstractMigrate);
>
> +# compared against remote end's minimum version
> +our $WS_TUNNEL_VERSION = 2;
> +
> sub fork_tunnel {
> my ($self, $ssh_forward_info) = @_;
>
> @@ -43,6 +46,35 @@ sub fork_tunnel {
> return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log);
> }
>
> +sub fork_websocket_tunnel {
> + my ($self, $storages, $bridges) = @_;
> +
> + my $remote = $self->{opts}->{remote};
> + my $conn = $remote->{conn};
> +
> + my $websocket_url = "https://$conn->{host}:$conn->{port}/api2/json/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnelwebsocket";
> + my $url = "/nodes/$self->{node}/qemu/$remote->{vmid}/mtunnel";
> +
> + my $tunnel_params = {
> + url => $websocket_url,
> + };
> +
> + my $storage_list = join(',', keys %$storages);
> + my $bridge_list = join(',', keys %$bridges);
> +
> + my $req_params = {
> + storages => $storage_list,
> + bridges => $bridge_list,
> + };
> +
> + my $log = sub {
> + my $line = shift;
> + $self->log('info', $line);
> + };
> +
> + return PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_params, $tunnel_params, $log);
> +}
> +
> # tunnel_info:
> # proto: unix (secure) or tcp (insecure/legacy compat)
> # addr: IP or UNIX socket path
> @@ -175,23 +207,34 @@ sub prepare {
> }
>
> my $vollist = PVE::QemuServer::get_vm_volumes($conf);
> +
> + my $storages = {};
> foreach my $volid (@$vollist) {
> my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
>
> - # check if storage is available on both nodes
> + # check if storage is available on source node
> my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
>
> my $targetsid = $sid;
> - # NOTE: we currently ignore shared source storages in mappings so skip here too for now
> - if (!$scfg->{shared}) {
> + # NOTE: local ignores shared mappings, remote maps them
> + if (!$scfg->{shared} || $self->{opts}->{remote}) {
> $targetsid = PVE::QemuServer::map_id($self->{opts}->{storagemap}, $sid);
> }
>
> - my $target_scfg = PVE::Storage::storage_check_enabled($storecfg, $targetsid, $self->{node});
> - my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
> + $storages->{$targetsid} = 1;
>
> - die "$volid: content type '$vtype' is not available on storage '$targetsid'\n"
> - if !$target_scfg->{content}->{$vtype};
> + if (!$self->{opts}->{remote}) {
> + # check if storage is available on target node
> + my $target_scfg = PVE::Storage::storage_check_enabled(
> + $storecfg,
> + $targetsid,
> + $self->{node},
> + );
> + my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
> +
> + die "$volid: content type '$vtype' is not available on storage '$targetsid'\n"
> + if !$target_scfg->{content}->{$vtype};
> + }
>
> if ($scfg->{shared}) {
> # PVE::Storage::activate_storage checks this for non-shared storages
> @@ -201,10 +244,27 @@ sub prepare {
> }
> }
>
> - # test ssh connection
> - my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
> - eval { $self->cmd_quiet($cmd); };
> - die "Can't connect to destination address using public key\n" if $@;
> + if ($self->{opts}->{remote}) {
> + # test & establish websocket connection
> + my $bridges = map_bridges($conf, $self->{opts}->{bridgemap}, 1);
> + my $tunnel = $self->fork_websocket_tunnel($storages, $bridges);
> + my $min_version = $tunnel->{version} - $tunnel->{age};
> + $self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION");
> + $self->log('info', "remote WS tunnel version: $tunnel->{version}");
> + $self->log('info', "minimum required WS tunnel version: $min_version");
> + die "Remote tunnel endpoint not compatible, upgrade required\n"
> + if $WS_TUNNEL_VERSION < $min_version;
> + die "Remote tunnel endpoint too old, upgrade required\n"
> + if $WS_TUNNEL_VERSION > $tunnel->{version};
> +
> + print "websocket tunnel started\n";
> + $self->{tunnel} = $tunnel;
> + } else {
> + # test ssh connection
> + my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];
> + eval { $self->cmd_quiet($cmd); };
> + die "Can't connect to destination address using public key\n" if $@;
> + }
>
> return $running;
> }
> @@ -242,7 +302,7 @@ sub scan_local_volumes {
> my @sids = PVE::Storage::storage_ids($storecfg);
> foreach my $storeid (@sids) {
> my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
> - next if $scfg->{shared};
> + next if $scfg->{shared} && !$self->{opts}->{remote};
> next if !PVE::Storage::storage_check_enabled($storecfg, $storeid, undef, 1);
>
> # get list from PVE::Storage (for unused volumes)
> @@ -251,19 +311,24 @@ sub scan_local_volumes {
> next if @{$dl->{$storeid}} == 0;
>
> my $targetsid = PVE::QemuServer::map_id($self->{opts}->{storagemap}, $storeid);
> - # check if storage is available on target node
> - my $target_scfg = PVE::Storage::storage_check_enabled(
> - $storecfg,
> - $targetsid,
> - $self->{node},
> - );
> -
> - die "content type 'images' is not available on storage '$targetsid'\n"
> - if !$target_scfg->{content}->{images};
> + my $bwlimit_sids = [$storeid];
> + if (!$self->{opts}->{remote}) {
> + # check if storage is available on target node
> + my $target_scfg = PVE::Storage::storage_check_enabled(
> + $storecfg,
> + $targetsid,
> + $self->{node},
> + );
> +
> + die "content type 'images' is not available on storage '$targetsid'\n"
> + if !$target_scfg->{content}->{images};
> +
> + push @$bwlimit_sids, $targetsid;
> + }
>
> my $bwlimit = PVE::Storage::get_bandwidth_limit(
> 'migration',
> - [$targetsid, $storeid],
> + $bwlimit_sids,
> $self->{opts}->{bwlimit},
> );
>
> @@ -319,14 +384,17 @@ sub scan_local_volumes {
> my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
>
> my $targetsid = $sid;
> - # NOTE: we currently ignore shared source storages in mappings so skip here too for now
> - if (!$scfg->{shared}) {
> + # NOTE: local ignores shared mappings, remote maps them
> + if (!$scfg->{shared} || $self->{opts}->{remote}) {
> $targetsid = PVE::QemuServer::map_id($self->{opts}->{storagemap}, $sid);
> }
>
> - PVE::Storage::storage_check_enabled($storecfg, $targetsid, $self->{node});
> + # check target storage on target node if intra-cluster migration
> + if (!$self->{opts}->{remote}) {
> + PVE::Storage::storage_check_enabled($storecfg, $targetsid, $self->{node});
>
> - return if $scfg->{shared};
> + return if $scfg->{shared};
> + }
>
> $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot';
> $local_volumes->{$volid}->{ref} = 'storage' if $attr->{is_unused};
> @@ -415,6 +483,9 @@ sub scan_local_volumes {
>
> my $migratable = $scfg->{type} =~ /^(?:dir|btrfs|zfspool|lvmthin|lvm)$/;
>
> + # TODO: what is this even here for?
> + $migratable = 1 if $self->{opts}->{remote};
> +
> die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n"
> if !$migratable;
>
> @@ -449,6 +520,10 @@ sub handle_replication {
> my $local_volumes = $self->{local_volumes};
>
> return if !$self->{replication_jobcfg};
> +
> + die "can't migrate VM with replicated volumes to remote cluster/node\n"
> + if $self->{opts}->{remote};
> +
> if ($self->{running}) {
>
> my $version = PVE::QemuServer::kvm_user_version();
> @@ -548,24 +623,51 @@ sub sync_offline_local_volumes {
> $self->log('info', "copying local disk images") if scalar(@volids);
>
> foreach my $volid (@volids) {
> - my $targetsid = $local_volumes->{$volid}->{targetsid};
> - my $bwlimit = $local_volumes->{$volid}->{bwlimit};
> - $bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps
> -
> - my $storage_migrate_opts = {
> - 'ratelimit_bps' => $bwlimit,
> - 'insecure' => $opts->{migration_type} eq 'insecure',
> - 'with_snapshots' => $local_volumes->{$volid}->{snapshots},
> - 'allow_rename' => !$local_volumes->{$volid}->{is_vmstate},
> - };
> + my $new_volid;
>
> - my $logfunc = sub { $self->log('info', $_[0]); };
> - my $new_volid = eval {
> - PVE::Storage::storage_migrate($storecfg, $volid, $self->{ssh_info},
> - $targetsid, $storage_migrate_opts, $logfunc);
> - };
> - if (my $err = $@) {
> - die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
> + my $opts = $self->{opts};
> + if ($opts->{remote}) {
> + my $log = sub {
> + my $line = shift;
> + $self->log('info', $line);
> + };
> +
> + $new_volid = PVE::StorageTunnel::storage_migrate(
> + $self->{tunnel},
> + $storecfg,
> + $volid,
> + $self->{vmid},
> + $opts->{remote}->{vmid},
> + $local_volumes->{$volid},
> + $log,
> + );
> + } else {
> + my $targetsid = $local_volumes->{$volid}->{targetsid};
> +
> + my $bwlimit = $local_volumes->{$volid}->{bwlimit};
> + $bwlimit = $bwlimit * 1024 if defined($bwlimit); # storage_migrate uses bps
> +
> + my $storage_migrate_opts = {
> + 'ratelimit_bps' => $bwlimit,
> + 'insecure' => $opts->{migration_type} eq 'insecure',
> + 'with_snapshots' => $local_volumes->{$volid}->{snapshots},
> + 'allow_rename' => !$local_volumes->{$volid}->{is_vmstate},
> + };
> +
> + my $logfunc = sub { $self->log('info', $_[0]); };
> + $new_volid = eval {
> + PVE::Storage::storage_migrate(
> + $storecfg,
> + $volid,
> + $self->{ssh_info},
> + $targetsid,
> + $storage_migrate_opts,
> + $logfunc,
> + );
> + };
> + if (my $err = $@) {
> + die "storage migration for '$volid' to storage '$targetsid' failed - $err\n";
> + }
> }
>
> $self->{volume_map}->{$volid} = $new_volid;
> @@ -581,6 +683,12 @@ sub sync_offline_local_volumes {
> sub cleanup_remotedisks {
> my ($self) = @_;
>
> + if ($self->{opts}->{remote}) {
> + PVE::Tunnel::finish_tunnel($self->{tunnel}, 1);
> + delete $self->{tunnel};
> + return;
> + }
> +
> my $local_volumes = $self->{local_volumes};
>
> foreach my $volid (values %{$self->{volume_map}}) {
> @@ -630,8 +738,100 @@ sub phase1 {
> $self->handle_replication($vmid);
>
> $self->sync_offline_local_volumes();
> + $self->phase1_remote($vmid) if $self->{opts}->{remote};
> };
>
> +sub map_bridges {
> + my ($conf, $map, $scan_only) = @_;
> +
> + my $bridges = {};
> +
> + foreach my $opt (keys %$conf) {
> + next if $opt !~ m/^net\d+$/;
> +
> + next if !$conf->{$opt};
> + my $d = PVE::QemuServer::parse_net($conf->{$opt});
> + next if !$d || !$d->{bridge};
> +
> + my $target_bridge = PVE::QemuServer::map_id($map, $d->{bridge});
> + $bridges->{$target_bridge}->{$opt} = $d->{bridge};
> +
> + next if $scan_only;
> +
> + $d->{bridge} = $target_bridge;
> + $conf->{$opt} = PVE::QemuServer::print_net($d);
> + }
> +
> + return $bridges;
> +}
> +
> +sub phase1_remote {
> + my ($self, $vmid) = @_;
> +
> + my $remote_conf = PVE::QemuConfig->load_config($vmid);
> + PVE::QemuConfig->update_volume_ids($remote_conf, $self->{volume_map});
> +
> + my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap});
> + for my $target (keys $bridges->%*) {
> + for my $nic (keys $bridges->{$target}->%*) {
> + $self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $target");
> + }
> + }
> +
> + my @online_local_volumes = $self->filter_local_volumes('online');
> +
> + my $storage_map = $self->{opts}->{storagemap};
> + $self->{nbd} = {};
> + PVE::QemuConfig->foreach_volume($remote_conf, sub {
> + my ($ds, $drive) = @_;
> +
> + # TODO eject CDROM?
> + return if PVE::QemuServer::drive_is_cdrom($drive);
> +
> + my $volid = $drive->{file};
> + return if !$volid;
> +
> + return if !grep { $_ eq $volid} @online_local_volumes;
> +
> + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
> + my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid);
> + my $source_format = PVE::QemuServer::qemu_img_format($scfg, $volname);
> +
> + # set by target cluster
> + my $oldvolid = delete $drive->{file};
> + delete $drive->{format};
> +
> + my $targetsid = PVE::QemuServer::map_id($storage_map, $storeid);
> +
> + my $params = {
> + format => $source_format,
> + storage => $targetsid,
> + drive => $drive,
> + };
> +
> + $self->log('info', "Allocating volume for drive '$ds' on remote storage '$targetsid'..");
> + my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'disk', $params);
> +
> + $self->log('info', "volume '$oldvolid' os '$res->{volid}' on the target\n");
s/os/is/
> + $remote_conf->{$ds} = $res->{drivestr};
> + $self->{nbd}->{$ds} = $res;
> + });
> +
> + my $conf_str = PVE::QemuServer::write_vm_config("remote", $remote_conf);
> +
> + # TODO expose in PVE::Firewall?
> + my $vm_fw_conf_path = "/etc/pve/firewall/$vmid.fw";
> + my $fw_conf_str;
> + $fw_conf_str = PVE::Tools::file_get_contents($vm_fw_conf_path)
> + if -e $vm_fw_conf_path;
> + my $params = {
> + conf => $conf_str,
> + 'firewall-config' => $fw_conf_str,
> + };
> +
> + PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params);
> +}
> +
> sub phase1_cleanup {
> my ($self, $vmid, $err) = @_;
>
> @@ -662,7 +862,6 @@ sub phase2_start_local_cluster {
> my $local_volumes = $self->{local_volumes};
> my @online_local_volumes = $self->filter_local_volumes('online');
>
> - $self->{storage_migration} = 1 if scalar(@online_local_volumes);
> my $start = $params->{start_params};
> my $migrate = $params->{migrate_opts};
>
> @@ -793,10 +992,34 @@ sub phase2_start_local_cluster {
> return ($tunnel_info, $spice_port);
> }
>
> +sub phase2_start_remote_cluster {
> + my ($self, $vmid, $params) = @_;
> +
> + die "insecure migration to remote cluster not implemented\n"
> + if $params->{migrate_opts}->{type} ne 'websocket';
> +
> + my $remote_vmid = $self->{opts}->{remote}->{vmid};
> +
> + my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 10, "start", $params);
10 seconds feels a bit short to me.
> +
> + foreach my $drive (keys %{$res->{drives}}) {
> + $self->{stopnbd} = 1;
> + $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr};
> + my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri};
> + die "unexpected NBD uri for '$drive': $nbd_uri\n"
> + if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
> +
> + $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri;
> + }
> +
> + return ($res->{migrate}, $res->{spice_port});
> +}
> +
> sub phase2 {
> my ($self, $vmid) = @_;
>
> my $conf = $self->{vmconf};
> + my $local_volumes = $self->{local_volumes};
>
> # version > 0 for unix socket support
> my $nbd_protocol_version = 1;
> @@ -828,10 +1051,39 @@ sub phase2 {
> },
> };
>
> - my ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
> + my ($tunnel_info, $spice_port);
> +
> + my @online_local_volumes = $self->filter_local_volumes('online');
> + $self->{storage_migration} = 1 if scalar(@online_local_volumes);
> +
> + if (my $remote = $self->{opts}->{remote}) {
> + my $remote_vmid = $remote->{vmid};
> + $params->{migrate_opts}->{remote_node} = $self->{node};
> + ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params);
> + die "only UNIX sockets are supported for remote migration\n"
> + if $tunnel_info->{proto} ne 'unix';
> +
> + my $remote_socket = $tunnel_info->{addr};
> + my $local_socket = $remote_socket;
> + $local_socket =~ s/$remote_vmid/$vmid/g;
> + $tunnel_info->{addr} = $local_socket;
> +
> + $self->log('info', "Setting up tunnel for '$local_socket'");
> + PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
> +
> + foreach my $remote_socket (@{$tunnel_info->{unix_sockets}}) {
> + my $local_socket = $remote_socket;
> + $local_socket =~ s/$remote_vmid/$vmid/g;
> + next if $self->{tunnel}->{forwarded}->{$local_socket};
> + $self->log('info', "Setting up tunnel for '$local_socket'");
> + PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
> + }
> + } else {
> + ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
>
> - $self->log('info', "start remote tunnel");
> - $self->start_remote_tunnel($tunnel_info);
> + $self->log('info', "start remote tunnel");
> + $self->start_remote_tunnel($tunnel_info);
> + }
>
> my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}";
> $migrate_uri .= ":$tunnel_info->{port}"
> @@ -841,8 +1093,6 @@ sub phase2 {
> $self->{storage_migration_jobs} = {};
> $self->log('info', "starting storage migration");
>
> - my @online_local_volumes = $self->filter_local_volumes('online');
> -
> die "The number of local disks does not match between the source and the destination.\n"
> if (scalar(keys %{$self->{target_drive}}) != scalar(@online_local_volumes));
> foreach my $drive (keys %{$self->{target_drive}}){
> @@ -915,7 +1165,7 @@ sub phase2 {
> };
> $self->log('info', "migrate-set-parameters error: $@") if $@;
>
> - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) {
> + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) {
> my $rpcenv = PVE::RPCEnvironment::get();
> my $authuser = $rpcenv->get_user();
>
> @@ -1112,11 +1362,15 @@ sub phase2_cleanup {
>
> my $nodename = PVE::INotify::nodename();
>
> - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
> - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> - if (my $err = $@) {
> - $self->log('err', $err);
> - $self->{errors} = 1;
> + if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) {
> + PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'stop');
> + } else {
> + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename];
> + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> + if (my $err = $@) {
> + $self->log('err', $err);
> + $self->{errors} = 1;
> + }
> }
>
> # cleanup after stopping, otherwise disks might be in-use by target VM!
> @@ -1149,7 +1403,7 @@ sub phase3_cleanup {
>
> my $tunnel = $self->{tunnel};
>
> - if ($self->{volume_map}) {
> + if ($self->{volume_map} && !$self->{opts}->{remote}) {
> my $target_drives = $self->{target_drive};
>
> # FIXME: for NBD storage migration we now only update the volid, and
> @@ -1165,27 +1419,34 @@ sub phase3_cleanup {
> }
>
> # transfer replication state before move config
> - $self->transfer_replication_state() if $self->{is_replicated};
> - PVE::QemuConfig->move_config_to_node($vmid, $self->{node});
> - $self->switch_replication_job_target() if $self->{is_replicated};
> + if (!$self->{opts}->{remote}) {
> + $self->transfer_replication_state() if $self->{is_replicated};
> + PVE::QemuConfig->move_config_to_node($vmid, $self->{node});
> + $self->switch_replication_job_target() if $self->{is_replicated};
> + }
>
> if ($self->{livemigration}) {
> if ($self->{stopnbd}) {
> $self->log('info', "stopping NBD storage migration server on target.");
> # stop nbd server on remote vm - requirement for resume since 2.9
> - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
> + if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) {
> + PVE::Tunnel::write_tunnel($tunnel, 30, 'nbdstop');
> + } else {
> + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid];
>
> - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> - if (my $err = $@) {
> - $self->log('err', $err);
> - $self->{errors} = 1;
> + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> + if (my $err = $@) {
> + $self->log('err', $err);
> + $self->{errors} = 1;
> + }
> }
> }
>
> # config moved and nbd server stopped - now we can resume vm on target
> if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) {
> + my $cmd = $tunnel->{version} == 1 ? "resume $vmid" : "resume";
> eval {
> - PVE::Tunnel::write_tunnel($tunnel, 30, "resume $vmid");
> + PVE::Tunnel::write_tunnel($tunnel, 30, $cmd);
> };
> if (my $err = $@) {
> $self->log('err', $err);
> @@ -1205,18 +1466,24 @@ sub phase3_cleanup {
> }
>
> if ($self->{storage_migration} && PVE::QemuServer::parse_guest_agent($conf)->{fstrim_cloned_disks} && $self->{running}) {
> - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim'];
> - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> + if ($self->{opts}->{remote}) {
> + PVE::Tunnel::write_tunnel($self->{tunnel}, 600, 'fstrim');
> + } else {
> + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'guest', 'cmd', $vmid, 'fstrim'];
> + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> + }
> }
> }
>
> # close tunnel on successful migration, on error phase2_cleanup closed it
> - if ($tunnel) {
> + if ($tunnel && $tunnel->{version} == 1) {
> eval { PVE::Tunnel::finish_tunnel($tunnel); };
> if (my $err = $@) {
> $self->log('err', $err);
> $self->{errors} = 1;
> }
> + $tunnel = undef;
> + delete $self->{tunnel};
> }
>
> eval {
> @@ -1254,6 +1521,9 @@ sub phase3_cleanup {
>
> # destroy local copies
> foreach my $volid (@not_replicated_volumes) {
> + # remote is cleaned up below
> + next if $self->{opts}->{remote};
> +
> eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
> if (my $err = $@) {
> $self->log('err', "removing local copy of '$volid' failed - $err");
> @@ -1263,8 +1533,19 @@ sub phase3_cleanup {
> }
>
> # clear migrate lock
> - my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
> - $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
> + if ($tunnel && $tunnel->{version} >= 2) {
> + PVE::Tunnel::write_tunnel($tunnel, 10, "unlock");
> +
> + PVE::Tunnel::finish_tunnel($tunnel);
> + } else {
> + my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ];
> + $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
> + }
> +
> + if ($self->{opts}->{remote} && $self->{opts}->{delete}) {
> + eval { PVE::QemuServer::destroy_vm($self->{storecfg}, $vmid, 1, undef, 0) };
> + warn "Failed to remove source VM - $@\n" if $@;
> + }
> }
>
> sub final_cleanup {
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 9971f2c..9490615 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -5425,7 +5425,10 @@ sub vm_start_nolock {
> my $defaults = load_defaults();
>
> # set environment variable useful inside network script
> - $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
> + # for remote migration the config is available on the target node!
> + if (!$migrate_opts->{remote_node}) {
> + $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
> + }
>
> PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
>
> @@ -5667,7 +5670,7 @@ sub vm_start_nolock {
>
> my $migrate_storage_uri;
> # nbd_protocol_version > 0 for unix socket support
> - if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
> + if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
> my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
> mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
> $migrate_storage_uri = "nbd:unix:$socket_path";
More information about the pve-devel
mailing list