[pve-devel] [PATCH] Use block storage migration for migration of KVM machines with local based storages

Mon Nov 3 13:29:04 CET 2014

I'm not sure either. We definitely need zero_blocks :)

On Mon, Nov 3, 2014 at 1:24 PM, Alexandre DERUMIER <aderumier at odiso.com>
wrote:

> Hi,
>
> I didn't known about the blk=> option in qmp migrate.
>
> Seem to be a lot easier than before.
>
>
> I'll try to test it this week.
>
>
> >>- enable migration caps: xbzrle and zero_blocks
>
> Why do you need xbzrle ? (I'm not sure It's 100% stable yet)
>
>
>
>
>
> ----- Mail original -----
>
> De: "Kamil Trzcinski" <ayufan at ayufan.eu>
> À: pve-devel at pve.proxmox.com
> Envoyé: Lundi 3 Novembre 2014 12:12:57
> Objet: [pve-devel] [PATCH] Use block storage migration for migration of
> KVM machines with local based storages
>
> - allow to migrate only VMs with either local or shared storage
> - for stopped VM start it for migration
> - allocate remote storage using ssh
> - enable migration caps: xbzrle and zero_blocks
>
> Signed-off-by: Kamil Trzcinski <ayufan at ayufan.eu>
> ---
> PVE/QemuMigrate.pm | 77
> ++++++++++++++++++++++++++++++++++++++++--------------
> PVE/QemuServer.pm | 4 +--
> 2 files changed, 60 insertions(+), 21 deletions(-)
>
> diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm
> index a49cdcc..8dcc8ab 100644
> --- a/PVE/QemuMigrate.pm
> +++ b/PVE/QemuMigrate.pm
> @@ -3,6 +3,7 @@ package PVE::QemuMigrate;
> use strict;
> use warnings;
> use PVE::AbstractMigrate;
> +use JSON;
> use IO::File;
> use IPC::Open2;
> use PVE::INotify;
> @@ -141,17 +142,14 @@ sub prepare {
> if (my $pid = PVE::QemuServer::check_running($vmid)) {
> die "cant migrate running VM without --online\n" if !$online;
> $running = $pid;
> + $self->{livemigration} = 1;
> $self->{forcemachine} = PVE::QemuServer::get_current_qemu_machine($vmid);
> - }
> -
> - if (my $loc_res = PVE::QemuServer::check_local_resources($conf, 1)) {
> - if ($self->{running} || !$self->{opts}->{force}) {
> - die "can't migrate VM which uses local devices\n";
> } else {
> - $self->log('info', "migrating VM which uses local devices");
> - }
> + $self->log('info', "starting VM $vmid on local node to perform
> migration");
> + PVE::QemuServer::vm_start($self->{storecfg}, $vmid, undef, 1, undef, 1);
> }
>
> + # do we need it? if we already do vm_start?
> # activate volumes
> my $vollist = PVE::QemuServer::get_vm_volumes($conf);
> PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
> @@ -163,13 +161,14 @@ sub prepare {
> eval { $self->cmd_quiet($cmd); };
> die "Can't connect to destination address using public key\n" if $@;
>
> - return $running;
> + # always perform online migration
> + return 1;
> }
>
> sub sync_disks {
> my ($self, $vmid) = @_;
>
> - $self->log('info', "copying disk images");
> + $self->log('info', "allocating disk images");
>
> my $conf = $self->{vmconf};
>
> @@ -238,8 +237,8 @@ sub sync_disks {
> $volhash->{$volid} = 1;
> });
>
> - if ($self->{running} && !$sharedvm) {
> - die "can't do online migration - VM uses local disks\n";
> + if (%$volhash) {
> + die "can't do migration - VM uses shared and local storage\n" if
> $sharedvm;
> }
>
> # do some checks first
> @@ -247,9 +246,6 @@ sub sync_disks {
> my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
> my $scfg = PVE::Storage::storage_config($self->{storecfg}, $sid);
>
> - die "can't migrate '$volid' - storagy type '$scfg->{type}' not
> supported\n"
> - if $scfg->{type} ne 'dir';
> -
> # if file, check if a backing file exist
> if (($scfg->{type} eq 'dir') && (!$sharedvm)) {
> my (undef, undef, undef, $parent) =
> PVE::Storage::volume_size_info($self->{storecfg}, $volid, 1);
> @@ -259,8 +255,8 @@ sub sync_disks {
>
> foreach my $volid (keys %$volhash) {
> my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
> + PVE::Storage::storage_migrate_alloc($self, $self->{storecfg}, $volid,
> $self->{nodeip}, $sid);
> push @{$self->{volumes}}, $volid;
> - PVE::Storage::storage_migrate($self->{storecfg}, $volid,
> $self->{nodeip}, $sid);
> }
> };
> die "Failed to sync data - $@" if $@;
> @@ -286,6 +282,15 @@ sub phase1_cleanup {
>
> $self->log('info', "aborting phase 1 - cleanup resources");
>
> + # always stop local VM if not doing livemigration
> + if(!$self->{livemigration}) {
> + eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); };
> + if (my $err = $@) {
> + $self->log('err', "stopping vm failed - $err");
> + $self->{errors} = 1;
> + }
> + }
> +
> my $conf = $self->{vmconf};
> delete $conf->{lock};
> eval { PVE::QemuServer::update_config_nolock($vmid, $conf, 1) };
> @@ -296,7 +301,8 @@ sub phase1_cleanup {
> if ($self->{volumes}) {
> foreach my $volid (@{$self->{volumes}}) {
> $self->log('err', "found stale volume copy '$volid' on node
> '$self->{node}'");
> - # fixme: try to remove ?
> + my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
> + PVE::Storage::storage_migrate_free($self, $self->{storecfg}, $volid,
> $self->{nodeip}, $sid);
> }
> }
> }
> @@ -359,7 +365,6 @@ sub phase2 {
>
> my $start = time();
> $self->log('info', "starting online/live migration on $raddr:$rport");
> - $self->{livemigration} = 1;
>
> # load_defaults
> my $defaults = PVE::QemuServer::load_defaults();
> @@ -417,7 +422,8 @@ sub phase2 {
> }
>
> eval {
> - PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate", uri =>
> "tcp:$raddr:$rport");
> + my $blk = $self->{volumes} ? JSON::true : JSON::false;
> + PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate", uri =>
> "tcp:$raddr:$rport", blk => $blk);
> };
> my $merr = $@;
> $self->log('info', "migrate uri => tcp:$raddr:$rport failed: $merr") if
> $merr;
> @@ -533,6 +539,11 @@ sub phase2_cleanup {
> };
> $self->log('info', "migrate_cancel error: $@") if $@;
>
> + # stop started VM
> + if (!$self->{livemigration}) {
> + eval{ PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1); }
> + }
> +
> my $conf = $self->{vmconf};
> delete $conf->{lock};
> eval { PVE::QemuServer::update_config_nolock($vmid, $conf, 1) };
> @@ -549,6 +560,14 @@ sub phase2_cleanup {
> $self->log('err', $err);
> $self->{errors} = 1;
> }
> +
> + if ($self->{volumes}) {
> + foreach my $volid (@{$self->{volumes}}) {
> + $self->log('err', "found stale volume copy '$volid' on node
> '$self->{node}'");
> + my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
> + PVE::Storage::storage_migrate_free($self, $self->{storecfg}, $volid,
> $self->{nodeip}, $sid);
> + }
> + }
> }
>
> sub phase3 {
> @@ -557,6 +576,13 @@ sub phase3 {
> my $volids = $self->{volumes};
> return if $self->{phase2errors};
>
> + # stop local VM so we can destroy local copies
> + eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); };
> + if (my $err = $@) {
> + $self->log('err', "stopping vm failed - $err");
> + $self->{errors} = 1;
> + }
> +
> # destroy local copies
> foreach my $volid (@$volids) {
> eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
> @@ -594,12 +620,25 @@ sub phase3_cleanup {
> $self->log('err', $err);
> $self->{errors} = 1;
> }
> + } else {
> + # now that config file is move, we can stop vm on target if not doing
> livemigrate
> + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock'];
> + eval{ PVE::Tools::run_command($cmd, outfunc => sub {},
> + errfunc => sub {
> + my $line = shift;
> + $self->log('err', $line);
> + });
> + };
> + if (my $err = $@) {
> + $self->log('err', $err);
> + $self->{errors} = 1;
> + }
> }
>
> eval {
>
> my $timer = 0;
> - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) &&
> $self->{running}) {
> + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) &&
> $self->{livemigration}) {
> $self->log('info', "Waiting for spice server migration");
> while (1) {
> my $res = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'query-spice');
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 98264d1..da4c2b7 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -3330,9 +3330,9 @@ sub set_migration_caps {
>
> my $enabled_cap = {
> "auto-converge" => 1,
> - "xbzrle" => 0,
> + "xbzrle" => 1,
> "x-rdma-pin-all" => 0,
> - "zero-blocks" => 0,
> + "zero-blocks" => 1,
> };
>
> my $supported_capabilities = vm_mon_cmd_nocheck($vmid,
> "query-migrate-capabilities");
> --
> 1.9.3 (Apple Git-50)
>
> _______________________________________________
> pve-devel mailing list
> pve-devel at pve.proxmox.com
> http://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>

-- 
Kamil Trzciński

ayufan at ayufan.eu
www.ayufan.eu
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.proxmox.com/pipermail/pve-devel/attachments/20141103/546fa02f/attachment.htm>