[pve-devel] [PATCH qemu-server v7 2/3] add QemuMigrateExternal.pm

Thomas Lamprecht t.lamprecht at proxmox.com
Tue Apr 30 17:10:46 CEST 2019


in general: please reorder patch 1 and 2, as the API entry depends on
the QemuMigrateExternal to exist.

Am 4/29/19 um 12:01 PM schrieb Alexandre Derumier:
> Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
> ---
>  PVE/Makefile               |   1 +
>  PVE/QemuMigrateExternal.pm | 340 +++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 341 insertions(+)
>  create mode 100644 PVE/QemuMigrateExternal.pm
> 
> diff --git a/PVE/Makefile b/PVE/Makefile
> index 2c800f6..0494cfb 100644
> --- a/PVE/Makefile
> +++ b/PVE/Makefile
> @@ -1,6 +1,7 @@
>  PERLSOURCE = 			\
>  	QemuServer.pm		\
>  	QemuMigrate.pm		\
> +	QemuMigrateExternal.pm	\
>  	QMPClient.pm		\
>  	QemuConfig.pm
>  
> diff --git a/PVE/QemuMigrateExternal.pm b/PVE/QemuMigrateExternal.pm
> new file mode 100644
> index 0000000..013ee4b
> --- /dev/null
> +++ b/PVE/QemuMigrateExternal.pm
> @@ -0,0 +1,340 @@
> +package PVE::QemuMigrateExternal;
> +
> +use strict;
> +use warnings;
> +use IO::File;
> +use IPC::Open2;
> +use POSIX qw( WNOHANG );
> +use PVE::INotify;
> +use PVE::Tools;
> +use PVE::Cluster;
> +use PVE::Storage;
> +use PVE::QemuServer;
> +use Time::HiRes qw( usleep );
> +use PVE::RPCEnvironment;
> +use Storable qw(dclone);
> +
> +use base qw(PVE::QemuMigrate);
> +
> +sub prepare {
> +    my ($self, $vmid) = @_;

from here..

> +
> +    my $online = $self->{opts}->{online};
> +
> +    $self->{storecfg} = PVE::Storage::config();
> +
> +    # test if VM exists
> +    my $conf = $self->{vmconf} = PVE::QemuConfig->load_config($vmid);
> +
> +    PVE::QemuConfig->check_lock($conf);
> +
> +    my $running = 0;
> +    if (my $pid = PVE::QemuServer::check_running($vmid)) {
> +	die "can't migrate running VM without --online\n" if !$online;
> +	$running = $pid;
> +
> +	$self->{forcemachine} = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
> +
> +    }
> +
> +    if (my $loc_res = PVE::QemuServer::check_local_resources($conf, 1)) {
> +	if ($self->{running} || !$self->{opts}->{force}) {
> +	    die "can't migrate VM which uses local devices\n";
> +	} else {
> +	    $self->log('info', "migrating VM which uses local devices");
> +	}
> +    }

...to here we are identical with the base QemuMigrate prepare

if we would move out the activate volume there and overwrite with an empty
method here, and use $self->vm_activate_volumes (better name welcomed) we may
save the whole prepare sub completely here

> +
> +    # test ssh connection
> +    push @{$self->{rem_ssh}}, '-i', $self->{opts}->{migration_external_sshkey};
> +    my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ];

this would be also great to get through a "class/plugin" method, e.g.:

$self->remote_cmd('/bin/true');

(just a suggestion for calling convention)

> +
> +    eval { $self->cmd_quiet($cmd); };
> +    die "Can't connect to destination address using public key\n" if $@;
> +
> +    find_targetvmid($self);

$self is "bless"ed with the calling class, so if you use $self->find_targetvmid();
in the QemuMigrate prepare, and just "overwrite" the find_targetvmid method here
in this perl module.

I hope I could clarify a bit what I'd like to see.. :) $self has the class inside,
as long as you call through it, the local overwritten method will be called, even
if the calling point is in the parent class.

I could try to bring up a skeleton for this to base off, but not sure if I come
to it this week.

> +
> +    generate_createvm_cmd($self, $conf);
> +        
> +    return 1;
> +
> +}
> +
> +sub phase1 {
> +    my ($self, $vmid) = @_;
> +
> +    $self->log('info', "starting migration of VM $vmid to node '$self->{node}' ($self->{nodeip})");
> +
> +    my $conf = $self->{vmconf};
> +
> +    # set migrate lock in config file
> +    $conf->{lock} = 'migrate';
> +    PVE::QemuConfig->write_config($vmid, $conf);
> +
> +    my $cmd = $self->{createcmd};
> +    
> +    # start create vm
> +    eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> +    if (my $err = $@) {
> +	$self->log('err', $err);
> +	$self->{errors} = 1;
> +	die $err;
> +    }
> +}
> +
> +sub phase1_cleanup {
> +    my ($self, $vmid, $err) = @_;
> +
> +    $self->log('info', "aborting phase 1 - cleanup resources");
> +
> +	my $targetvmid = $self->{opts}->{targetvmid} ? $self->{opts}->{targetvmid} : $vmid;
> +	
> +    unlock_vm($self, $vmid);
> +
> +    cleanup_remote_vm($self, $targetvmid);
> +}
> +
> +
> +sub phase2 {
> +    my ($self, $vmid) = @_;
> +
> +    my $targetvmid = $self->{opts}->{targetvmid} ? $self->{opts}->{targetvmid} : $vmid;
> +
> +    my $nodename = PVE::INotify::nodename();
> +        
> +    $self->log('info', "starting VM $vmid on remote node '$self->{node}'");
> +
> +    my $migration_type = 'secure';
> +
> +    my $cmd = generate_migrate_start_cmd($self, $targetvmid, $nodename, $migration_type);
> +    
> +    my ($raddr, $rport, $ruri, $spice_port, $spice_ticket) = PVE::QemuMigrate::find_remote_ports($self, $targetvmid, $cmd);
> +
> +    PVE::QemuMigrate::start_remote_tunnel($self, $nodename, $migration_type, $raddr, $rport, $ruri);
> +
> +    livemigrate_storage($self, $vmid);
> +    
> +    PVE::QemuMigrate::livemigrate($self, $vmid, $ruri, $spice_port);
> +    
> +}
> +
> +sub phase2_cleanup {
> +    my ($self, $vmid, $err) = @_;
> +
> +    return if !$self->{errors};
> +
> +    my $targetvmid = $self->{opts}->{targetvmid} ? $self->{opts}->{targetvmid} : $vmid;
> +
> +    $self->{phase2errors} = 1;
> +
> +    $self->log('info', "aborting phase 2 - cleanup resources");
> +
> +    PVE::QemuMigrate::cancel_migrate($self, $vmid);
> +
> +    PVE::QemuMigrate::unlock_vm($self, $vmid);
> +
> +    cleanup_remote_vm($self, $targetvmid);
> +
> +    if ($self->{tunnel}) {
> +	eval { PVE::QemuMigrate::finish_tunnel($self, $self->{tunnel});  };
> +	if (my $err = $@) {
> +	    $self->log('err', $err);
> +	    $self->{errors} = 1;
> +	}
> +    }
> +}
> +
> +
> +sub phase3_cleanup {
> +    my ($self, $vmid, $err) = @_;
> +
> +    return if $self->{phase2errors};
> +        
> +    my $targetvmid = $self->{opts}->{targetvmid} ? $self->{opts}->{targetvmid} : $vmid;
> +
> +    finish_block_jobs($self, $vmid);
> +        
> +    PVE::QemuMigrate::finish_livemigration($self, $targetvmid);
> +    
> +    PVE::QemuMigrate::finish_spice_migration($self, $vmid);
> +
> +    PVE::QemuMigrate::stop_local_vm($self, $vmid);
> +
> +    # clear migrate lock
> +    my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $targetvmid ];
> +    $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");
> +}
> +
> +sub find_targetvmid {
> +    my ($self) = @_;
> +    
> +    if (!$self->{opts}->{targetvmid}) {
> +	#get remote nextvmid
> +	eval {
> +	    my $cmd = [@{$self->{rem_ssh}}, 'pvesh', 'get', '/cluster/nextid'];
> +	    PVE::Tools::run_command($cmd, outfunc => sub {
> +		my $line = shift;
> +		if ($line =~ m/^(\d+)/) {
> +		    $self->{opts}->{targetvmid} = $line;
> +		}
> +	    });
> +	};
> +	if (my $err = $@) {
> +	    $self->log('err', $err);
> +	    $self->{errors} = 1;
> +	    die $err;
> +	}
> +
> +	die "can't find the next free vmid on remote cluster\n" if !$self->{opts}->{targetvmid};
> +    }
> +
> +}
> +
> +sub generate_createvm_cmd {
> +    my ($self, $conf) = @_;
> +    
> +    my $cmd = [@{$self->{rem_ssh}}, 'qm', 'create', $self->{opts}->{targetvmid}];
> +
> +    my $target_conf = dclone $conf;
> +
> +    foreach my $opt (keys %{$target_conf}) {
> +	next if $opt =~ m/^(pending|snapshots|digest|parent)/;
> +	next if $opt =~ m/^(ide|scsi|virtio)(\d+)/;
> +
> +	if ($opt =~ m/^(net)(\d+)/ && $self->{opts}->{$opt}) {
> +	    my $oldnet = PVE::QemuServer::parse_net($target_conf->{$opt});
> +	    my $newnet = PVE::QemuServer::parse_net($self->{opts}->{$opt});
> +	    foreach my $newnet_opt (keys %$newnet) {
> +		next if $newnet_opt =~ m/^(model|macaddr|queues)$/;
> +		$oldnet->{$newnet_opt} = $newnet->{$newnet_opt};
> +	    }
> +	    $target_conf->{$opt} = PVE::QemuServer::print_net($oldnet);
> +	}
> +
> +	die "can't migrate unused disk. please remove it before migrate\n" if $opt =~ m/^(unused)(\d+)/;
> +	push @$cmd , "-$opt", PVE::Tools::shellquote($target_conf->{$opt});
> +    }
> +
> +    PVE::QemuServer::foreach_drive($target_conf, sub {
> +	my ($ds, $drive) = @_;
> +
> +	if (PVE::QemuServer::drive_is_cdrom($drive, 1)) {
> +	    push @$cmd , "-$ds", PVE::Tools::shellquote($target_conf->{$ds});
> +	    return;
> +	}
> +
> +	my $volid = $drive->{file};
> +	return if !$volid;
> +
> +	my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
> +	return if !$sid;
> +	my $size = PVE::Storage::volume_size_info($self->{storecfg}, $volid, 5);
> +	die "can't get size\n" if !$size;
> +	$size = $size/1024/1024/1024;
> +	my $targetsid = $self->{opts}->{targetstorage} ? $self->{opts}->{targetstorage} : $sid;
> +
> +	my $data = { %$drive };
> +	delete $data->{$_} for qw(index interface file size);
> +	my $drive_conf = "$targetsid:$size";
> +	foreach my $drive_opt (keys %{$data}) {
> +	    $drive_conf .= ",$drive_opt=$data->{$drive_opt}";
> +	}
> +
> +	push @$cmd , "-$ds", PVE::Tools::shellquote($drive_conf);
> +    });
> +
> +    push @$cmd , '-lock', 'migrate';
> +
> +    $self->{createcmd} = $cmd;
> +}
> +
> +sub generate_migrate_start_cmd {
> +    my ($self, $vmid, $nodename, $migration_type) = @_;
> +
> +    my $cmd = [@{$self->{rem_ssh}}];
> +
> +    push @$cmd , 'qm', 'start', $vmid, '--skiplock';
> +
> +    push @$cmd, '--external_migration';
> +        
> +    push @$cmd, '--migration_type', $migration_type;
> +
> +    push @$cmd, '--migration_network', $self->{opts}->{migration_network}
> +      if $self->{opts}->{migration_network};
> +
> +    if ($migration_type eq 'insecure') {
> +	push @$cmd, '--stateuri', 'tcp';
> +    } else {
> +	push @$cmd, '--stateuri', 'unix';
> +    }
> +
> +    if ($self->{forcemachine}) {
> +	push @$cmd, '--machine', $self->{forcemachine};
> +    }
> +
> +    if ($self->{online_local_volumes}) {
> +	push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1');
> +    }
> +    
> +    return $cmd;
> +}
> +
> +sub livemigrate_storage {
> +    my ($self, $vmid) = @_;
> +
> +    my $conf = $self->{vmconf};
> +
> +	$self->{storage_migration} = 1;
> +	$self->{storage_migration_jobs} = {};
> +	$self->log('info', "starting storage migration");
> +	foreach my $drive (keys %{$self->{target_drive}}){
> +	    my $target = $self->{target_drive}->{$drive};
> +	    my $nbd_uri = $target->{nbd_uri};
> +	    #bandwith, source only
> +	    my $source_sid = PVE::Storage::Plugin::parse_volume_id($conf->{$drive});
> +	    my $bwlimit = PVE::Storage::get_bandwidth_limit('migrate', [$source_sid, undef], $self->{opts}->{bwlimit});
> +	    $self->log('info', "$drive: start migration to $nbd_uri");
> +	    PVE::QemuServer::qemu_drive_mirror($vmid, $drive, $nbd_uri, $vmid, undef, $self->{storage_migration_jobs}, 1, undef, $bwlimit);
> +	}
> +
> +}
> +
> +sub finish_block_jobs {
> +    my ($self, $vmid) = @_;
> +
> +    my $conf = $self->{vmconf};
> +
> +    if ($self->{storage_migration}) {
> +	# finish block-job
> +	eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}); };
> +
> +	if (my $err = $@) {
> +	    eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $self->{storage_migration_jobs}) };
> +	    eval { PVE::QemuMigrate::cleanup_remotedisks($self) };
> +	    die "Failed to completed storage migration\n";
> +	}
> +    }    
> +}
> +
> +
> +sub cleanup_remote_vm {
> +    my ($self, $vmid) = @_;
> +    
> +    my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock'];
> +
> +    eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> +    if (my $err = $@) {
> +        $self->log('err', $err);
> +        $self->{errors} = 1;
> +    }
> +
> +    $cmd = [@{$self->{rem_ssh}}, 'qm', 'destroy', $vmid, '--skiplock'];
> +
> +    eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) };
> +    if (my $err = $@) {
> +	$self->log('err', $err);
> +	$self->{errors} = 1;
> +    }
> +}
> +
> +1;
> 





More information about the pve-devel mailing list