[pve-devel] [PATCH] Use block storage migration for migration of KVM machines with local based storages

Alexandre DERUMIER aderumier at odiso.com
Mon Nov 3 13:24:10 CET 2014


Hi,

I didn't known about the blk=> option in qmp migrate.

Seem to be a lot easier than before.


I'll try to test it this week.


>>- enable migration caps: xbzrle and zero_blocks 

Why do you need xbzrle ? (I'm not sure It's 100% stable yet)





----- Mail original ----- 

De: "Kamil Trzcinski" <ayufan at ayufan.eu> 
À: pve-devel at pve.proxmox.com 
Envoyé: Lundi 3 Novembre 2014 12:12:57 
Objet: [pve-devel] [PATCH] Use block storage migration for migration of KVM machines with local based storages 

- allow to migrate only VMs with either local or shared storage 
- for stopped VM start it for migration 
- allocate remote storage using ssh 
- enable migration caps: xbzrle and zero_blocks 

Signed-off-by: Kamil Trzcinski <ayufan at ayufan.eu> 
--- 
PVE/QemuMigrate.pm | 77 ++++++++++++++++++++++++++++++++++++++++-------------- 
PVE/QemuServer.pm | 4 +-- 
2 files changed, 60 insertions(+), 21 deletions(-) 

diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm 
index a49cdcc..8dcc8ab 100644 
--- a/PVE/QemuMigrate.pm 
+++ b/PVE/QemuMigrate.pm 
@@ -3,6 +3,7 @@ package PVE::QemuMigrate; 
use strict; 
use warnings; 
use PVE::AbstractMigrate; 
+use JSON; 
use IO::File; 
use IPC::Open2; 
use PVE::INotify; 
@@ -141,17 +142,14 @@ sub prepare { 
if (my $pid = PVE::QemuServer::check_running($vmid)) { 
die "cant migrate running VM without --online\n" if !$online; 
$running = $pid; 
+ $self->{livemigration} = 1; 
$self->{forcemachine} = PVE::QemuServer::get_current_qemu_machine($vmid); 
- } 
- 
- if (my $loc_res = PVE::QemuServer::check_local_resources($conf, 1)) { 
- if ($self->{running} || !$self->{opts}->{force}) { 
- die "can't migrate VM which uses local devices\n"; 
} else { 
- $self->log('info', "migrating VM which uses local devices"); 
- } 
+ $self->log('info', "starting VM $vmid on local node to perform migration"); 
+ PVE::QemuServer::vm_start($self->{storecfg}, $vmid, undef, 1, undef, 1); 
} 

+ # do we need it? if we already do vm_start? 
# activate volumes 
my $vollist = PVE::QemuServer::get_vm_volumes($conf); 
PVE::Storage::activate_volumes($self->{storecfg}, $vollist); 
@@ -163,13 +161,14 @@ sub prepare { 
eval { $self->cmd_quiet($cmd); }; 
die "Can't connect to destination address using public key\n" if $@; 

- return $running; 
+ # always perform online migration 
+ return 1; 
} 

sub sync_disks { 
my ($self, $vmid) = @_; 

- $self->log('info', "copying disk images"); 
+ $self->log('info', "allocating disk images"); 

my $conf = $self->{vmconf}; 

@@ -238,8 +237,8 @@ sub sync_disks { 
$volhash->{$volid} = 1; 
}); 

- if ($self->{running} && !$sharedvm) { 
- die "can't do online migration - VM uses local disks\n"; 
+ if (%$volhash) { 
+ die "can't do migration - VM uses shared and local storage\n" if $sharedvm; 
} 

# do some checks first 
@@ -247,9 +246,6 @@ sub sync_disks { 
my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); 
my $scfg = PVE::Storage::storage_config($self->{storecfg}, $sid); 

- die "can't migrate '$volid' - storagy type '$scfg->{type}' not supported\n" 
- if $scfg->{type} ne 'dir'; 
- 
# if file, check if a backing file exist 
if (($scfg->{type} eq 'dir') && (!$sharedvm)) { 
my (undef, undef, undef, $parent) = PVE::Storage::volume_size_info($self->{storecfg}, $volid, 1); 
@@ -259,8 +255,8 @@ sub sync_disks { 

foreach my $volid (keys %$volhash) { 
my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); 
+ PVE::Storage::storage_migrate_alloc($self, $self->{storecfg}, $volid, $self->{nodeip}, $sid); 
push @{$self->{volumes}}, $volid; 
- PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{nodeip}, $sid); 
} 
}; 
die "Failed to sync data - $@" if $@; 
@@ -286,6 +282,15 @@ sub phase1_cleanup { 

$self->log('info', "aborting phase 1 - cleanup resources"); 

+ # always stop local VM if not doing livemigration 
+ if(!$self->{livemigration}) { 
+ eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); }; 
+ if (my $err = $@) { 
+ $self->log('err', "stopping vm failed - $err"); 
+ $self->{errors} = 1; 
+ } 
+ } 
+ 
my $conf = $self->{vmconf}; 
delete $conf->{lock}; 
eval { PVE::QemuServer::update_config_nolock($vmid, $conf, 1) }; 
@@ -296,7 +301,8 @@ sub phase1_cleanup { 
if ($self->{volumes}) { 
foreach my $volid (@{$self->{volumes}}) { 
$self->log('err', "found stale volume copy '$volid' on node '$self->{node}'"); 
- # fixme: try to remove ? 
+ my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); 
+ PVE::Storage::storage_migrate_free($self, $self->{storecfg}, $volid, $self->{nodeip}, $sid); 
} 
} 
} 
@@ -359,7 +365,6 @@ sub phase2 { 

my $start = time(); 
$self->log('info', "starting online/live migration on $raddr:$rport"); 
- $self->{livemigration} = 1; 

# load_defaults 
my $defaults = PVE::QemuServer::load_defaults(); 
@@ -417,7 +422,8 @@ sub phase2 { 
} 

eval { 
- PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate", uri => "tcp:$raddr:$rport"); 
+ my $blk = $self->{volumes} ? JSON::true : JSON::false; 
+ PVE::QemuServer::vm_mon_cmd_nocheck($vmid, "migrate", uri => "tcp:$raddr:$rport", blk => $blk); 
}; 
my $merr = $@; 
$self->log('info', "migrate uri => tcp:$raddr:$rport failed: $merr") if $merr; 
@@ -533,6 +539,11 @@ sub phase2_cleanup { 
}; 
$self->log('info', "migrate_cancel error: $@") if $@; 

+ # stop started VM 
+ if (!$self->{livemigration}) { 
+ eval{ PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1); } 
+ } 
+ 
my $conf = $self->{vmconf}; 
delete $conf->{lock}; 
eval { PVE::QemuServer::update_config_nolock($vmid, $conf, 1) }; 
@@ -549,6 +560,14 @@ sub phase2_cleanup { 
$self->log('err', $err); 
$self->{errors} = 1; 
} 
+ 
+ if ($self->{volumes}) { 
+ foreach my $volid (@{$self->{volumes}}) { 
+ $self->log('err', "found stale volume copy '$volid' on node '$self->{node}'"); 
+ my ($sid, $volname) = PVE::Storage::parse_volume_id($volid); 
+ PVE::Storage::storage_migrate_free($self, $self->{storecfg}, $volid, $self->{nodeip}, $sid); 
+ } 
+ } 
} 

sub phase3 { 
@@ -557,6 +576,13 @@ sub phase3 { 
my $volids = $self->{volumes}; 
return if $self->{phase2errors}; 

+ # stop local VM so we can destroy local copies 
+ eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); }; 
+ if (my $err = $@) { 
+ $self->log('err', "stopping vm failed - $err"); 
+ $self->{errors} = 1; 
+ } 
+ 
# destroy local copies 
foreach my $volid (@$volids) { 
eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; 
@@ -594,12 +620,25 @@ sub phase3_cleanup { 
$self->log('err', $err); 
$self->{errors} = 1; 
} 
+ } else { 
+ # now that config file is move, we can stop vm on target if not doing livemigrate 
+ my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock']; 
+ eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, 
+ errfunc => sub { 
+ my $line = shift; 
+ $self->log('err', $line); 
+ }); 
+ }; 
+ if (my $err = $@) { 
+ $self->log('err', $err); 
+ $self->{errors} = 1; 
+ } 
} 

eval { 

my $timer = 0; 
- if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) { 
+ if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{livemigration}) { 
$self->log('info', "Waiting for spice server migration"); 
while (1) { 
my $res = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'query-spice'); 
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm 
index 98264d1..da4c2b7 100644 
--- a/PVE/QemuServer.pm 
+++ b/PVE/QemuServer.pm 
@@ -3330,9 +3330,9 @@ sub set_migration_caps { 

my $enabled_cap = { 
"auto-converge" => 1, 
- "xbzrle" => 0, 
+ "xbzrle" => 1, 
"x-rdma-pin-all" => 0, 
- "zero-blocks" => 0, 
+ "zero-blocks" => 1, 
}; 

my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities"); 
-- 
1.9.3 (Apple Git-50) 

_______________________________________________ 
pve-devel mailing list 
pve-devel at pve.proxmox.com 
http://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel 


More information about the pve-devel mailing list