[pve-devel] [RFC pve-storage 5/7] This patch will include storage asynchronous replication.
Wolfgang Bumiller
w.bumiller at proxmox.com
Thu Apr 13 16:02:39 CEST 2017
On Wed, Apr 12, 2017 at 12:41:24PM +0200, Wolfgang Link wrote:
> It is possible to synchronise a volume to an other node in a defined interval.
> So if a node fail there will be an copy of the volumes from a VM
> on an other node.
> With this copy it is possible to start the VM on this node.
> ---
> Makefile | 12 +-
> PVE/API2/Makefile | 1 +
> PVE/API2/StorageReplication.pm | 56 +++++
> PVE/CLI/Makefile | 2 +-
> PVE/CLI/pvesr.pm | 178 ++++++++++++++
> PVE/Makefile | 1 +
> PVE/ReplicationTools.pm | 546 +++++++++++++++++++++++++++++++++++++++++
> pvesr | 8 +
> 8 files changed, 801 insertions(+), 3 deletions(-)
> create mode 100644 PVE/API2/StorageReplication.pm
> create mode 100644 PVE/CLI/pvesr.pm
> create mode 100644 PVE/ReplicationTools.pm
> create mode 100644 pvesr
>
> diff --git a/Makefile b/Makefile
> index 57500e3..2aab912 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -33,15 +33,23 @@ pvesm.bash-completion:
> perl -I. -T -e "use PVE::CLI::pvesm; PVE::CLI::pvesm->generate_bash_completions();" >$@.tmp
> mv $@.tmp $@
>
> +pvesr.bash-completion:
> + perl -I. -T -e "use PVE::CLI::pvesr; PVE::CLI::pvesr->generate_bash_completions();" >$@.tmp
> + mv $@.tmp $@
> +
> .PHONY: install
> -install: pvesm.1 pvesm.bash-completion
> +install: pvesm.1 pvesm.bash-completion pvesr.bash-completion
> install -d ${DESTDIR}${SBINDIR}
> install -m 0755 pvesm ${DESTDIR}${SBINDIR}
> + install -m 0755 pvesr ${DESTDIR}${SBINDIR}
> make -C PVE install
> + install -d ${DESTDIR}/var/lib/pve-replica
> install -d ${DESTDIR}/usr/share/man/man1
> install -m 0644 pvesm.1 ${DESTDIR}/usr/share/man/man1/
> gzip -9 -n ${DESTDIR}/usr/share/man/man1/pvesm.1
> install -m 0644 -D pvesm.bash-completion ${DESTDIR}${BASHCOMPLDIR}/pvesm
> + install -m 0644 -D pvesr.bash-completion ${DESTDIR}${BASHCOMPLDIR}/pverepm
> +
>
> .PHONY: deb
> deb: ${DEB}
> @@ -65,7 +73,7 @@ ${DEB}:
> .PHONY: clean
> clean:
> make cleanup-docgen
> - rm -rf debian *.deb ${PACKAGE}-*.tar.gz dist *.1 *.tmp pvesm.bash-completion
> + rm -rf debian *.deb ${PACKAGE}-*.tar.gz dist *.1 *.tmp pvesm.bash-completion pvesr.bash-completion
> find . -name '*~' -exec rm {} ';'
>
> .PHONY: distclean
> diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile
> index 7b7226e..621221d 100644
> --- a/PVE/API2/Makefile
> +++ b/PVE/API2/Makefile
> @@ -3,4 +3,5 @@
> .PHONY: install
> install:
> install -D -m 0644 Disks.pm ${DESTDIR}${PERLDIR}/PVE/API2/Disks.pm
> + install -D -m 0644 StorageReplication.pm ${DESTDIR}${PERLDIR}/PVE/API2/StorageReplication.pm
> make -C Storage install
> diff --git a/PVE/API2/StorageReplication.pm b/PVE/API2/StorageReplication.pm
> new file mode 100644
> index 0000000..da70b15
> --- /dev/null
> +++ b/PVE/API2/StorageReplication.pm
> @@ -0,0 +1,56 @@
> +package PVE::API2::StorageReplication;
> +
> +use warnings;
> +use strict;
> +
> +use Data::Dumper qw(Dumper);
> +
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::ReplicationTools;
> +use PVE::INotify;
> +
> +use PVE::RESTHandler;
> +
> +use base qw(PVE::RESTHandler);
> +
> +__PACKAGE__->register_method ({
> + name => 'list',
> + path => 'list',
> + method => 'GET',
> + description => "List of all replications",
> + permissions => {
> + user => 'all',
> + },
> + protected => 1,
> + proxyto => 'node',
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + nodes => get_standard_option('pve-node-list' ,
> + {description => "Notes where the jobs is located.",
> + optional => 1}),
> + 'json' => {
> + optional => 1,
> + type => 'boolean',
> + description => "Output in JSON format.",
> + },
> + },
> + },
> + returns => { type => 'object' },
> + code => sub {
> + my ($param) = @_;
> +
> + if ($param->{nodes}) {
> + foreach my $node (PVE::Tools::split_list($param->{nodes})) {
> + die "Node: $node does not exists.\n" if
> + !PVE::Cluster::check_node_exists($node);
> + }
Indentation/control flow seem to be broken here.
> +
> + my $nodes = $param->{nodes} ?
> + $param->{nodes} : PVE::INotify::nodename();
> +
> + return PVE::ReplicationTools::get_all_jobs($nodes, $param->{json});
> +}}});
> +
> +1;
> diff --git a/PVE/CLI/Makefile b/PVE/CLI/Makefile
> index 6c6e258..3d6f96d 100644
> --- a/PVE/CLI/Makefile
> +++ b/PVE/CLI/Makefile
> @@ -1,4 +1,4 @@
> -SOURCES=pvesm.pm
> +SOURCES=pvesm.pm pvesr.pm
>
> .PHONY: install
> install: ${SOURCES}
> diff --git a/PVE/CLI/pvesr.pm b/PVE/CLI/pvesr.pm
> new file mode 100644
> index 0000000..c1f3fb4
> --- /dev/null
> +++ b/PVE/CLI/pvesr.pm
> @@ -0,0 +1,178 @@
> +package PVE::CLI::pvesr;
> +
> +use strict;
> +use warnings;
> +
> +use PVE::API2::StorageReplication;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::INotify;
> +use PVE::RPCEnvironment;
> +use PVE::Tools qw(extract_param);
> +use PVE::SafeSyslog;
> +use PVE::CLIHandler;
> +
> +use base qw(PVE::CLIHandler);
> +
> +my $nodename = PVE::INotify::nodename();
> +
> +my $MAX_FAIL = 3;
> +
> +sub setup_environment {
> + PVE::RPCEnvironment->setup_default_cli_env();
> +}
> +
> +my $time2timestr = sub {
> + my ($time) = @_;
> +
> + my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) =
> + localtime($time);
> + my $datestamp = sprintf ("%04d-%02d-%02d_%02d:%02d:%02d",
> + $year+1900, $mon+1, $mday, $hour, $min, $sec);
> +
> + #Return in format YYYY-MM-DD_hh:mm:ss
This could be:
strftime("%Y-%m-%d_%H:%M:%S", localtime($time)
> + return $datestamp;
> +};
> +
> +my $print_list = sub {
> + my ($conf) = @_;
> +
> + if ($conf->{json}) {
> + delete $conf->{json};
> + print JSON::encode_json($conf);
> + } else {
> +
> + printf("%-10s%-20s%-20s%-5s%-10s%-5s\n",
You should add spaces between the fields here and in the printf list
below otherwise if a field is >= the allowed length it'll be stuck to
the neighbor.
> + "VMID", "DEST", "LAST SYNC","IVAL", "STATE", "FAIL");
> +
> + foreach my $vmid (sort keys %$conf) {
> +
> + my $timestr = &$time2timestr($conf->{$vmid}->{lastsync});
> +
> + printf("%-10s", $vmid);
> + printf("%-20s", $conf->{$vmid}->{tnode});
> + printf("%-20s", $timestr);
> + printf("%-5s", $conf->{$vmid}->{interval});
> + printf("%-10s", $conf->{$vmid}->{state});
> + printf("%-10s\n", $conf->{$vmid}->{fail});
> + }
> + }
> +};
> +
> +my $get_replica_list = sub {
> +
> + my $jobs = PVE::ReplicationTools::read_state();
> + my $list = {};
> +
> + foreach my $vmid (keys %$jobs) {
> +
> + my $lastsync = $jobs->{$vmid}->{lastsync};
Please add a helper variable like
my $job = $jobs->{$vmid};
rather than using $jobs->{$vmid}->{xyz} so many times.
> +
> + # interval in min
> + my $interval = $jobs->{$vmid}->{interval};
> + my $now = time();
> + my $fail = $jobs->{$vmid}->{fail};
> +
> + my $synctime = $lastsync + $interval * 60;
> +
> + $synctime += $interval * $fail if $fail > 0;
AFAICT fail cannot be negative, so the condition is superfluous.
Unless it can be undefined, then defined($fail) is still necessary with
the condition in place.
> +
> + if ($now > $synctime && ($jobs->{$vmid}->{state} eq 'ok'
> + || ($jobs->{$vmid}->{state} eq 'ok') && ($fail <= $MAX_FAIL)) ){
I'd prefer to see more parenthesis here.
This is:
A && B || C && D
While && binds more strongly than || I'd still perfer
(A && B) || (C && D)
for clarity.
> +
> + $list->{$synctime} = $vmid;
> + }
> + }
> +
> + return $list;
> +};
> +
> +my $replicate_vms = sub {
> + my ($list) = @_;
> +
> + my $oldest_rep = 0;
> +
> + while (%$list) {
> +
> + foreach my $synctime (keys %$list) {
> +
> + if ($oldest_rep == 0 || $oldest_rep > $synctime) {
> + $oldest_rep = $synctime;
> + }
> + }
How about starting the function with:
my @sorted_times = reverse sort keys %$list;
and replacing the `while (%$list)` with a
foreach my $synctime (@sorted_times)
You could then drop the `delete` and you wouldn't be modifying the
caller's $list which makes for a much nicer interface.
> +
> + eval {
> + PVE::ReplicationTools::sync_guest($list->{$oldest_rep});
> + };
> + if (my $err = $@) {
> + syslog ('err', $err );
> + }
> + delete $list->{$oldest_rep};
> + $oldest_rep = 0;
> +
> + }
> +};
> +
> +__PACKAGE__->register_method ({
> + name => 'run',
> + path => 'run',
> + method => 'POST',
> + description => "This method will run by the systemd-timer and sync all jobs",
> + permissions => {
> + description => {
> + check => ['perm', '/', [ 'Sys.Console' ]],
> + },
> + },
> + protected => 1,
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + },
> + },
> + returns => { type => 'null' },
> + code => sub {
> +
> + my $list = &$get_replica_list();
> + &$replicate_vms($list);
> +
> + return undef;
> + }});
> +
> +__PACKAGE__->register_method ({
> + name => 'destroyjob',
> + path => 'destroyjob',
> + method => 'DELETE',
> + description => "Destroy an async replication job",
> + permissions => {
> + description => {
> + check => ['perm', '/storage', ['Datastore.Allocate']],
> + },
> + },
> + protected => 1,
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + vmid => {
> + type => 'string', format => 'pve-vmid',
> + description => "The VMID of the guest.",
> + completion => \&PVE::Cluster::complete_local_vmid,
> + },
> + },
> + },
> + returns => { type => 'null' },
> + code => sub {
> + my ($param) = @_;
> +
> + my $vmid = extract_param($param, 'vmid');
> +
> + PVE::ReplicationTools::destroy_replica($vmid);
> +
> + }});
> +
> +our $cmddef = {
> + list => [ 'PVE::API2::StorageReplication' , 'list' , [], { node => $nodename },
> + $print_list],
> + run => [ __PACKAGE__ , 'run'],
> + destroyjob => [ __PACKAGE__ , 'destroyjob', ['vmid']],
> +};
> +
> +1;
> diff --git a/PVE/Makefile b/PVE/Makefile
> index ae2bd35..c4075d1 100644
> --- a/PVE/Makefile
> +++ b/PVE/Makefile
> @@ -3,6 +3,7 @@
> .PHONY: install
> install:
> install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm
> + install -D -m 0644 ReplicationTools.pm ${DESTDIR}${PERLDIR}/PVE/ReplicationTools.pm
> install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
> make -C Storage install
> make -C API2 install
> diff --git a/PVE/ReplicationTools.pm b/PVE/ReplicationTools.pm
> new file mode 100644
> index 0000000..57d9c7a
> --- /dev/null
> +++ b/PVE/ReplicationTools.pm
> @@ -0,0 +1,546 @@
> +package PVE::ReplicationTools;
> +
> +use warnings;
> +use strict;
> +
> +use PVE::Tools qw(run_command);
> +use PVE::Cluster;
> +use PVE::QemuConfig;
> +use PVE::LXC::Config;
> +use PVE::LXC;
> +use PVE::Storage;
> +use Time::Local;
> +use JSON;
> +use Data::Dumper qw(Dumper);
> +
> +my $STATE_DIR = '/var/lib/pve-replica';
> +my $STATE_FILE = "/pve-replica.state";
> +my $STATE_PATH = $STATE_DIR.$STATE_FILE;
> +
> +PVE::Cluster::cfs_update;
> +my $local_node = PVE::INotify::nodename();
> +
> +my $cluster_nodes;
> +
> +my $get_guestconfig = sub {
> + my ($vmid) = @_;
> +
> + my $vms = PVE::Cluster::get_vmlist();
> +
> + my $type = $vms->{ids}->{$vmid}->{type};
> +
> + my $guestconf;
> + my $running;
> +
> + if ($type =~ m/^qemu$/) {
> + $guestconf = PVE::QemuConfig->load_config($vmid);
> + $running = PVE::QemuServer::check_running($vmid);
> + } elsif ($type =~ m/^lxc$/) {
> + $guestconf = PVE::LXC::Config->load_config($vmid);
> + $running = PVE::LXC::check_running($vmid);
> + }
> +
> + return ($guestconf, $type, $running);
> +};
> +
> +sub write_state {
> + my ($state) = @_;
> +
> + # create dir if not exists
> + if (!-e $STATE_DIR) {
> + mkdir $STATE_DIR if !(-d $STATE_DIR);
> + }
Too many checks. You're ignoring mkdir's error anyway and let
file_set_contents() fail instead, so a simple unconditional
mkdir $STATE_DIR;
should be fine.
> +
> + PVE::Tools::file_set_contents($STATE_PATH, JSON::encode_json($state));
> +}
> +
> +sub read_state {
> +
> + return {} if !(-e $STATE_PATH);
> +
> + my $raw = PVE::Tools::file_get_contents($STATE_PATH);
> +
> + return {} if $raw eq '';
> + return JSON::decode_json($raw);
> +}
> +
> +my $get_nodelist = sub {
You don't seem to be using this function?
Also, PVE::Cluster::get_nodelist() already uses a cache AFAIK.
(PVE::Cluster::cfs_update() would update it).
> + my ($update) = @_;
> +
> + if (defined($update) || !defined($cluster_nodes)) {
> +
> + $cluster_nodes = PVE::Cluster::get_nodelist();
> + }
> +
> + return $cluster_nodes;
> +};
> +
> +sub get_node_ip {
> + my ($nodename) = @_;
> +
> + my $remoteip = PVE::Cluster::remote_node_ip($nodename, 1);
> +
> + my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
> + if (my $network = $dc_conf->{storage_migration_network}) {
> +
> + my $cmd = ['ssh', '-o', 'Batchmode=yes', "root\@$remoteip", '--'
> + ,'pvecm', 'mtunnel', '--get_migration_ip',
> + '--migration_network', $network];
> +
> + PVE::Tools::run_command($cmd, outfunc => sub {
> + my $line = shift;
> +
> + if ($line =~ m/^ip: '($PVE::Tools::IPRE)'$/) {
> + $remoteip = $1;
> + }
> + });
> + }
> + return $remoteip;
> +}
> +
> +sub get_all_jobs {
> + my ($nodes, $json) = @_;
> +
> + my @nodelist = PVE::Tools::split_list($nodes);
> +
> + my $vms = PVE::Cluster::get_vmlist();
> + my $state = read_state();
> + my $jobs = {};
> +
> + my $outfunc = sub {
> + my $line = shift;
> +
> + $jobs = JSON::decode_json($line);
This replaces the $jobs contents for each line.
> + };
> +
> + foreach my $node (@nodelist) {
> + if (!($local_node eq $node)) {
use: ($local_node ne $node)
> +
> + my $ip = get_node_ip($node);
> + $ip = [$ip] if Net::IP::ip_is_ipv6($ip);
> +
> + my @cmd = ('ssh', '-o', 'Batchmode=yes', "root\@$ip", '--',
> + 'pvesr', 'list', '--json');
> +
> + run_command([@cmd], outfunc=>$outfunc)
See $outfunc: $jobs is completely replaced for every non-local node.
What is the intention? Given that the else branch "merges" $vmids into
the jobs hash.
> +
> + } else {
> +
> + foreach my $vmid (keys %{$vms->{ids}}) {
> +
> + next if !($vms->{ids}->{$vmid}->{node} eq $local_node);
> + next if !defined($state->{$vmid});
> +
> + $jobs->{$vmid}->{limit} = $state->{$vmid}->{limit};
Please add $state->{$vmid} and $jobs->{$vmid} holding helper variables
for this whole loop body.
Also consider:
$jobs->{$vmid} = { %{$state->{$vmid}} };
or can there be different contents in $jobs and $state? Then maybe
consider:
foreach (qw(interval tnode lastsync state fail)) {
$jobs->{$vmid}->{$_} = $state->{$vmid}->{$_};
}
(Not necessary though)
> + $jobs->{$vmid}->{interval} = $state->{$vmid}->{interval};
> + $jobs->{$vmid}->{tnode} = $state->{$vmid}->{tnode};
> + $jobs->{$vmid}->{lastsync} = $state->{$vmid}->{lastsync};
> + $jobs->{$vmid}->{state} = $state->{$vmid}->{state};
> + $jobs->{$vmid}->{fail} = $state->{$vmid}->{fail};
> + }
> +
> + }
> + }
> +
> + $jobs->{json} = 1 if $json;
> + return ($jobs);
> +}
> +
> +sub sync_guest {
> + my ($vmid, $param) = @_;
> +
> + my $jobs = read_state();
> +
> + my ($guest_conf, $vm_type, $running) = &$get_guestconfig($vmid);
> + my $qga = 0;
> +
> + my $tnode = $jobs->{$vmid}->{tnode};
Maybe also a shortcut here:
my $job = $jobs->{$vmid};
> +
> + if ($vm_type eq "qemu" && defined($guest_conf->{agent}) ) {
> + $qga = PVE::QemuServer::qga_check_running($vmid)
> + if PVE::QemuServer::check_running($vmid);
if $running? Since get_guestconfig() already calls check_running().
> + }
> +
> + # will not die if a disk is not syncable
> + my $disks = get_syncable_guestdisks($guest_conf, $vm_type);
> +
> + # check if all nodes have the storage availible
> + my $storage_config = PVE::Storage::config();
> + foreach my $volid (keys %$disks) {
> + my ($storeid) = PVE::Storage::parse_volume_id($volid);
> +
> + die "Storage not availible on node: $tnode\n"
> + if !$storage_config->{ids}->{$storeid}->{nodes}->{$tnode};
> + die "Storage not availible on node: $local_node\n"
> + if !$storage_config->{ids}->{$storeid}->{nodes}->{$local_node};
> +
> + }
> +
> + my $limit = $param->{limit};
> + $limit = $guest_conf->{replica_rate_limit}
> + if (!defined($param->{limit}) &&
> + defined($guest_conf->{replica_rate_limit}));
my $limit = $param->{limit} // $guest_conf->{replica_rate_limit};
Or our more common:
my $limit = $param->{limit};
$limit = $guest_conf->{replica_rate_limit} if !defined($limit);
(Checking if the $guest_conf->{replica_rate_limit} is defined is
pointless since if it isn't, $limit will be just as undefined as it was
before ;-) )
> +
> + my $snap_time = time();
> +
> + die "Not vailid synctime" if $jobs->{$vmid}->{lastsync} !~ m/^(\d+)$/;
"Invalid" - but maybe explain a little more?
> + my $lastsync = $1;
> + my $incremental_snap = $lastsync ? "replica_$lastsync" : undef;
> +
> + # freeze filesystem for data consistency
> + if ($qga == 1 ) {
if ($qga) ?
(Several times below as well)
> + print "Freeze guest filesystem\n";
> +
> + eval {
> + PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze");
> + };
> + }
> +
> + my $snapname = "replica_$snap_time";
> +
> + my $disks_status = {};
> + $disks_status->{snapname} = $snapname;
my $disks_status = { snapname => $snapname };
> +
> + my $sync_job = sub {
> +
> + # make snapshot of all volumes
> + foreach my $volid (keys %$disks) {
> +
> + eval {
> + PVE::Storage::volume_snapshot($storage_config, $volid, $snapname);
> + };
> +
> + if (my $err = $@) {
> + if ($qga == 1) {
> + print "Unfreeze guest filesystem\n";
> + eval {
> + PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw");
> + };
Does this warn if it fails? Otherwise: warn $@ if $@;
> + }
> + cleanup_snapshot($disks_status, $snapname, $storage_config, $running);
> + $jobs->{$vmid}->{state} = 'err';
> + write_state($jobs);
> +
> + die $err;
> + }
> +
> + $disks_status->{$volid}->{snapshot} = 1;
> + }
> +
> + if ($qga == 1) {
> + print "Unfreeze guest filesystem\n";
> + eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
As above.
> + }
> +
> + my $ip = get_node_ip($tnode);
> +
> + foreach my $volid (keys %$disks) {
> +
> + eval {
> + PVE::Storage::volume_send($storage_config, $volid, $snapname,
> + $ip, $incremental_snap,
> + $param->{verbose}, $limit);
> + $jobs->{$vmid}->{fail} = 0;
> + };
> +
> + if (my $err = $@) {
> + cleanup_snapshot($disks_status, $snapname, $storage_config, $running, $ip);
> + $jobs->{$vmid}->{fail}++;
> + $jobs->{$vmid}->{state} = 'err' if $jobs->{$vmid}->{fail} > 3;
> + write_state($jobs);
> + die "$err";
> + }
> +
> + $disks_status->{$volid}->{synced} = 1;
> + }
> +
> + # delet old snapshot if exists
> + cleanup_snapshot($disks_status, $snapname, $storage_config, $running, $ip, $lastsync) if
> + $jobs->{$vmid}->{lastsync} ne '0';
> +
> + $jobs->{$vmid}->{lastsync} = $snap_time;
> + write_state($jobs);
> + };
> +
> + PVE::Tools::lock_file_full($STATE_PATH, 60, 0 , $sync_job);
> + die $@ if $@;
Another reminder that the lock_file_full() interface needs to change.
> +
> + return $snap_time;
> +}
> +
> +sub get_snapshots {
> + my ($vol, $prefix, $nodes) = @_;
> +
> + my $plugin = $vol->{plugin};
> + return $plugin->get_snapshots($vol, $prefix, $nodes);
> +}
> +
> +sub send_image {
> + my ($vol, $param, $ip, $all_snaps_in_delta, $alter_path) = @_;
> +
> + my $plugin = $vol->{plugin};
> + $plugin->send_image($vol, $param, $ip, $all_snaps_in_delta, $alter_path);
> +}
> +
> +sub job_enable {
> + my ($vmid, $no_sync) = @_;
> +
> + my $update_state = sub {
> + my ($state) = @_;
> +
> + my $jobs = read_state();
> +
> + my ($config) = &$get_guestconfig($vmid);
> + my $param = {};
> +
> + $jobs->{$vmid}->{interval} = $config->{replica_interval} ?
> + $config->{replica_interval} : 15;
$jobs->{$vmid}->{interval} = $config->{replica_interval} || 15;
> +
> + if ( defined($config->{replica_limit})) {
> + $jobs->{$vmid}->{limit} = $config->{replica_limit};
> + $param->{limit} = $config->{replica_limit};
> + }
> +
> + die "Replica Target must be set\n" if !defined($config->{replica_target});
> + $jobs->{$vmid}->{tnode} = $config->{replica_target};
> +
> + $jobs->{$vmid}->{fail} = 0;
> + if (!defined($jobs->{$vmid}->{lastsync})) {
> +
> + if ( my $lastsync = get_lastsync($vmid)) {
> + $jobs->{$vmid}->{lastsync} = $lastsync;
> + } else {
> + $jobs->{$vmid}->{lastsync} = 0;
> + }
> + }
> +
> + $param->{verbose} = 1;
> +
> + $jobs->{$vmid}->{state} = 'ok';
> + write_state($jobs);
> +
> + eval{
> + sync_guest($vmid, $param) if !defined($no_sync);
> + };
> + if (my $err = $@) {
> + $jobs->{$vmid}->{state} = 'error';
> + write_state($jobs);
> + die $err;
> + }
> + };
> +
> + PVE::Tools::lock_file_full($STATE_PATH, 5, 0 , $update_state);
> + die $@ if $@;
> +}
> +
> +sub job_disable {
> + my ($vmid) = @_;
> +
> + my $update_state = sub {
> +
> + my $jobs = read_state();
> +
> + if (defined($jobs->{$vmid})) {
> + $jobs->{$vmid}->{state} = 'off';
> + write_state($jobs);
> + } else {
> + print "No replica service for $vmid\n";
> + }
> + };
> +
> + PVE::Tools::lock_file_full($STATE_PATH, 5, 0 , $update_state);
> + die $@ if $@;
> +}
> +
> +sub job_remove {
> + my ($vmid) = @_;
> +
> + my $update_state = sub {
> +
> + my $jobs = read_state();
> +
> + if (defined($jobs->{$vmid})) {
> + delete($jobs->{$vmid});
> + write_state($jobs);
> + } else {
> + print "No replica service for $vmid\n";
> + }
> + };
> +
> + PVE::Tools::lock_file_full($STATE_PATH, 5, 0 , $update_state);
> + die $@ if $@;
> +}
> +
> +sub get_syncable_guestdisks {
> + my ($config, $vm_type, $running) = @_;
> +
> + my $syncable_disks = {};
> +
> + my $cfg = PVE::Storage::config();
> +
> + my $warnings = 0;
> + my $func = sub {
> + my ($id, $volume) = @_;
> +
> + my $volname;
> + if ($vm_type eq 'qemu') {
> + $volname = $volume->{file};
> + } else {
> + $volname = $volume->{volume};
> + }
> +
> + if( PVE::Storage::volume_has_feature($cfg, 'replicate', $volname , undef, $running)) {
> + $syncable_disks->{$volname} = 1;
> + } else {
> + warn "Can't sync Volume: $volname"
missing \n
> + if (!defined($volume->{replica}) || $volume->{replica});
> + $warnings = 1;
> + }
> + };
> +
> + if ($vm_type eq 'qemu') {
> + PVE::QemuServer::foreach_drive($config, $func);
> + } elsif ($vm_type eq 'lxc') {
> + PVE::LXC::Config->foreach_mountpoint($config, $func);
> + } else {
> + die "Unknown VM Type: $vm_type";
> + }
> +
> + return wantarray ? ($warnings, $syncable_disks) : $syncable_disks;
> +}
> +
> +sub destroy_all_snapshots {
> + my ($vmid, $prefix, $node) = @_;
> +
> + my $ip = defined($node) ? get_node_ip($node) : undef;
> +
> + my ($guest_conf, $vm_type, $running) = &$get_guestconfig($vmid);
> +
> + my $disks = get_syncable_guestdisks($guest_conf, $vm_type);
> + my $cfg = PVE::Storage::config();
> +
> + my $snapshots = {};
> + foreach my $volid (keys %$disks) {
> + $snapshots->{$volid} =
> + PVE::Storage::volume_snapshot_list($cfg, $volid, 'rep', $node, $ip);
The patch implementing this function is missing.
And you're using 'rep' here, and 'replica' in the other calls to it.
(A regex as parameter might be a nicer interface btw.)
> + }
> +
> + foreach my $volid (keys %$snapshots) {
> +
> + if (defined($prefix)) {
> + foreach my $snap (@{$snapshots->{$volid}}) {
> + PVE::Storage::volume_snapshot_delete($cfg, $volid, $snap, $running, $ip);
> + }
> + } else {
> + if ($ip) {
> +
> + my $cmd = ['ssh', '-o', 'Batchmode=yes', "root\@$ip", '--'
> + ,'pvesm', 'free', $volid];
> + PVE::Tools::run_command($cmd);
> + } else {
> + PVE::Storage::vdisk_free($cfg, $volid);
> + }
> + }
> + }
> +
> +}
> +
> +sub cleanup_snapshot {
> + my ($disks, $snapname, $cfg, $running, $ip, $lastsync_snap) = @_;
> +
> + if ($lastsync_snap) {
> + $snapname = "replica_$lastsync_snap";
> + }
> +
> + foreach my $volid (keys %$disks) {
> + next if $volid eq "snapname";
> +
> + if (defined($lastsync_snap) || $disks->{$volid}->{synced}) {
> + PVE::Storage::volume_snapshot_delete($cfg, $volid, $snapname, $running, $ip);
> + }
> +
> + if (defined($lastsync_snap) || $disks->{$volid}->{snapshot}) {
> + PVE::Storage::volume_snapshot_delete($cfg, $volid, $snapname, $running);
> + }
> + }
> +}
> +
> +sub destroy_replica {
> + my ($vmid) = @_;
> +
> + my $code = sub {
> +
> + my $jobs = read_state();
> +
> + return if !defined($jobs->{$vmid});
> +
> + my ($guest_conf, $vm_type) = &$get_guestconfig($vmid);
> +
> + destroy_all_snapshots($vmid, 'replica');
> + destroy_all_snapshots($vmid, undef, $guest_conf->{replica_target});
> +
> + delete($jobs->{$vmid});
> +
> + delete($guest_conf->{replica_rate_limit});
> + delete($guest_conf->{replica_rate_interval});
> + delete($guest_conf->{replica_target});
> + delete($guest_conf->{replica});
> +
> + if ($vm_type eq 'qemu') {
> + PVE::QemuConfig->write_config($vmid, $guest_conf);
> + } else {
> + PVE::LXC::Config->write_config($vmid, $guest_conf);
> + }
> + write_state($jobs);
> + };
> +
> + PVE::Tools::lock_file_full($STATE_PATH, 30, 0 , $code);
> + die $@ if $@;
> +}
> +
> +sub get_lastsync {
> + my ($vmid) = @_;
> +
> + my ($conf, $vm_type) = &$get_guestconfig($vmid);
> +
> + my $sync_vol = get_syncable_guestdisks($conf, $vm_type);
> + my $cfg = PVE::Storage::config();
> +
> + my $time;
> + foreach my $volid (keys %$sync_vol) {
> + my $list =
> + PVE::Storage::volume_snapshot_list($cfg, $volid, 'replica', $local_node);
> +
> + if (my $tmp_snap = shift @$list) {
> + $tmp_snap =~ m/^replica_(\d+)$/;
> + die "snapshots are not coherent"
missing \n
> + if defined($time) && !($time eq $1);
> + $time = $1;
> + }
> + }
> +
> + return $time;
> +}
> +
> +sub get_last_replica_snap {
> + my ($volid) = @_;
> +
> + my $cfg = PVE::Storage::config();
> + my $list = PVE::Storage::volume_snapshot_list($cfg, $volid, 'replica', $local_node);
> +
> + my $snap = shift @$list;
> + die "Not a valid Sanpshot" if $snap !~ m/replica_(\d+)/;
With a regex interface in volume_snapshot_list you wouldn't need that
check. (Also you could have included the underscore in the prefix ;-) )
> + return $snap;
> +}
> +
> +sub is_guest_volumes_syncable {
'is' vs 'are' vs 'check' (the latter could have a $noerr and die() if
not $noerr - the callers qemu-server/pve-container wouldn't need to
contain the error message then)
> + my ($conf, $vm_type) = @_;
> +
> + my ($warnings, $disks) = get_syncable_guestdisks($conf, $vm_type);
get_syncable_guestdisks() should probably have a flag for whether to
actually call `warn()`, since a function called
'is_guest_volumes_syncable()' should probably not spew warnings when
there are ones which aren't syncable.
> +
> + return undef if $warnings || !%$disks;
> +
> + return 1;
> +}
> +
> +1;
> diff --git a/pvesr b/pvesr
The (number of) 2-letter-suffix pvexy commands bug(s) me
a lot.
> new file mode 100644
> index 0000000..ffcf84c
> --- /dev/null
> +++ b/pvesr
> @@ -0,0 +1,8 @@
> +#!/usr/bin/perl
> +
> +use strict;
> +use warnings;
> +
> +use PVE::CLI::pvesr;
> +
> +PVE::CLI::pvesr->run_cli_handler();
> --
> 2.1.4
More information about the pve-devel
mailing list