[pve-devel] [pve-container] LXC implement migration on replicated storage

[Wolfgang Link]

---
 src/PVE/LXC/Migrate.pm | 76 ++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 70 insertions(+), 6 deletions(-)

diff --git a/src/PVE/LXC/Migrate.pm b/src/PVE/LXC/Migrate.pm
index 2f0244b..cfca82c 100644
--- a/src/PVE/LXC/Migrate.pm
+++ b/src/PVE/LXC/Migrate.pm
@@ -10,6 +10,10 @@ use PVE::INotify;
 use PVE::Cluster;
 use PVE::Storage;
 use PVE::LXC;
+use PVE::LXC::Config;
+use PVE::ReplicationConfig;
+use PVE::ReplicationState;
+use JSON;
 
 use base qw(PVE::AbstractMigrate);
 
@@ -264,12 +268,42 @@ sub phase1 {
 	die "can't migrate CT - check log\n";
     }
 
+
+    my $rep_cfg = PVE::ReplicationConfig->new();
+    my $jobid;
+    my $last_replica;
+
+    if ($jobid = PVE::ReplicationConfig::get_jobid($rep_cfg, $vmid, $self->{node}) ) {
+       my $stateobj = PVE::ReplicationState::read_state();
+       my $state = PVE::ReplicationState::extract_job_state($stateobj, $rep_cfg->{ids}->{$jobid});
+       $last_replica = PVE::ReplicationState::replication_snapshot_name($jobid, $state->{last_sync});
+    }
+
     foreach my $volid (keys %$volhash) {
 	my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
 	push @{$self->{volumes}}, $volid;
-	PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{ssh_info}, $sid);
+	PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{ssh_info}, $sid, undef, $last_replica);
+    }
+
+
+    # set new replica_target if we migrate to replica target.
+
+    if ($jobid) {
+
+       my $update_target = sub {
+
+           if ($rep_cfg->{ids}->{$jobid}->{target} eq $self->{node}) {
+               $self->log('info', "change replica target to Node: $self->{opts}->{node}");
+               $rep_cfg->{ids}->{$jobid}->{target} = $self->{opts}->{node};
+               $rep_cfg->write();
+           }
+
+       };
+
+       PVE::ReplicationConfig::lock($update_target);
     }
 
+
     my $conffile = PVE::LXC::Config->config_file($vmid);
     my $newconffile = PVE::LXC::Config->config_file($vmid, $self->{node});
 
@@ -312,13 +346,19 @@ sub phase3 {
 
     my $volids = $self->{volumes};
 
+    my $rep_cfg = PVE::ReplicationConfig->new();
+    my $replica = PVE::ReplicationConfig::check_for_existing_jobs($rep_cfg, $vmid, 1);
+    my $jobid =  PVE::ReplicationConfig::get_jobid($rep_cfg, $vmid, $self->{opts}->{node});
     # destroy local copies
     foreach my $volid (@$volids) {
-	eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
-	if (my $err = $@) {
-	    $self->log('err', "removing local copy of '$volid' failed - $err");
-	    $self->{errors} = 1;
-	    last if $err =~ /^interrupted by signal$/;
+	# do not destroy if new target is local_host
+	if (!$jobid) {
+	    eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
+	    if (my $err = $@) {
+		$self->log('err', "removing local copy of '$volid' failed - $err");
+		$self->{errors} = 1;
+		last if $err =~ /^interrupted by signal$/;
+	    }
 	}
     }
 }
@@ -341,6 +381,30 @@ sub final_cleanup {
 	$self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");	
     }
 
+    # transfer replication state to migration target node.
+    my $rep_cfg = PVE::ReplicationConfig->new();
+    my $new_target = $self->{opts}->{node};
+    if (PVE::ReplicationConfig::get_jobid($rep_cfg, $vmid, $new_target)) {
+	my $stateobj = PVE::ReplicationState::read_state();
+
+	my $old_target =  $self->{node};
+
+	my $vm_state = {};
+	foreach my $key (keys %{$stateobj->{$vmid}}) {
+	    if ($key =~ /^(.*\/)\Q$old_target\E$/) {
+		$vm_state->{"$1$new_target"} = $stateobj->{$vmid}->{$key};
+	    } else {
+		$vm_state->{$key} = $stateobj->{$vmid}->{$key};
+	    }
+	}
+
+	# This have to be double quoted when it run it over ssh.
+	my $state = PVE::Tools::shellquote(encode_json $vm_state);
+
+	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $vmid, $state];
+	$self->cmd_logerr($cmd, errmsg => "failed to activate replica");
+    }
+
     # in restart mode, we start the container on the target node
     # after migration
     if ($self->{opts}->{restart} && $self->{was_running}) {
-- 
2.11.0