[pve-devel] [PATCH 2/2] New Backup Strategy
Jeff Moskow
jeff at rtr.com
Fri Jan 17 15:53:49 CET 2014
Signed-off-by: Jeff Moskow <jeff at rtr.com>
---
PVE/API2/Backup.pm | 4 +-
PVE/VZDump.pm | 136 ++++++++++++++++++++++++++--
www/manager/Makefile | 1 +
www/manager/dc/Backup.js | 21 ++++-
www/manager/form/BackupStrategySelector.js | 17 ++++
www/manager/grid/BackupView.js | 1 +
www/manager/window/Backup.js | 7 ++
7 files changed, 173 insertions(+), 14 deletions(-)
diff --git a/PVE/API2/Backup.pm b/PVE/API2/Backup.pm
index ddcdf57..c26c8d3 100644
--- a/PVE/API2/Backup.pm
+++ b/PVE/API2/Backup.pm
@@ -78,7 +78,7 @@ sub parse_dow {
return $res;
};
-my $vzdump_propetries = {
+my $vzdump_properties = {
additionalProperties => 0,
properties => PVE::VZDump::json_config_properties({}),
};
@@ -112,7 +112,7 @@ sub parse_vzdump_cron_config {
die "unable to parse day of week '$dow' in '$filename'\n" if !$dowhash;
my $args = PVE::Tools::split_args($param);
- my $opts = PVE::JSONSchema::get_options($vzdump_propetries, $args, 'vmid');
+ my $opts = PVE::JSONSchema::get_options($vzdump_properties, $args, 'vmid');
$opts->{id} = "$digest:$jid";
$jid++;
diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 77fd211..daf919d 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -10,6 +10,7 @@ use IO::Select;
use IPC::Open3;
use POSIX qw(strftime);
use File::Path;
+use File::ReadBackwards;
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::Cluster qw(cfs_read_file);
@@ -91,6 +92,7 @@ sub storage_info {
PVE::Storage::activate_storage($cfg, $storage);
return {
+ name => $storage,
dumpdir => PVE::Storage::get_backup_dir($cfg, $storage),
maxfiles => $scfg->{maxfiles},
};
@@ -494,8 +496,11 @@ sub new {
$opts->{storage} = 'local';
}
- if ($opts->{storage}) {
- my $info = storage_info ($opts->{storage});
+ # test first storage option if multiples are supplied, we'll pick the right one at runtime
+ (my $storage_test = $opts->{storage}) =~ s/,.*//;
+ debugmsg('debug', $opts->{storage}.' -- '.$storage_test, undef, 1);
+ if ($storage_test) {
+ my $info = storage_info ($storage_test);
$opts->{dumpdir} = $info->{dumpdir};
$maxfiles = $info->{maxfiles} if !defined($maxfiles) && defined($info->{maxfiles});
} elsif ($opts->{dumpdir}) {
@@ -706,13 +711,73 @@ sub exec_backup_task {
eval {
die "unable to find VM '$vmid'\n" if !$plugin;
+ # test if VM is running
+ my ($running, $status_text) = $plugin->vm_status ($vmid);
+
my $vmtype = $plugin->type();
+ my $bkname = "vzdump-$vmtype-$vmid";
+
+ # find oldest and newest existing backup on each storage
+ # find any storage w/o any backup
+ my $fill_candidate->{date} = $vmstarttime; # looking for storage that's most out of date
+ my $replace_candidate->{date} = $vmstarttime; # looking for storage that has the oldest backup
+ my $newest = 0;
+ my $nobackup = undef;
+ my $storage;
+ my %backup_dates;
+ my $last_change = &last_change_entry($vmid);
+ for $storage (split(',',$opts->{storage})) {
+ my $info = storage_info($storage);
+ my @files = glob($info->{dumpdir}.'/'.$bkname.'*');
+ my $used = 0; # how many times is this storage used for this VM
+ my $this_newest = 0;
+ my $this_oldest = $vmstarttime;
+ foreach (@files) {
+ next if /\.log$/;
+ my $mtime = (stat($_))[9];
+ $this_newest = $mtime if $mtime > $this_newest;
+ $this_oldest = $mtime if $mtime < $this_oldest;
+ $used++;
+ }
+ $nobackup = $info if ! $used;
+ $newest = $this_newest if $this_newest > $newest;
+ $fill_candidate->{date} = $this_newest, $fill_candidate->{storage} = $info if $this_newest < $fill_candidate->{date};
+ $replace_candidate->{date} = $this_oldest, $replace_candidate->{storage} = $info if $this_oldest < $replace_candidate->{date};
+ }
+ if (defined($nobackup)) {
+ $storage = $nobackup;
+ } elsif ($opts->{strategy} =~ /^(distribute|aggressive|safe)/) {
+ $storage = $fill_candidate->{storage};
+ } else {
+ # option is 'always'
+ $storage = $replace_candidate->{storage};
+ }
+ if (!$running && $opts->{strategy} !~ /^(always|distribute)/) {
+ my $skip = 0;
+ # skip if all storages in use and up to date
+ $skip++ if ($opts->{strategy} eq 'safe' && !defined($nobackup) && $last_change < $fill_candidate->{date});
+ # skip if we have at least one backup that's newer than the last change time
+ $skip++ if ($opts->{strategy} eq 'aggressive' && $last_change < $newest);
+ if ($skip) {
+ debugmsg ('info', "VM $vmid is unchanged, skipping backup", $logfd);
+ $task->{tarfile} = "VM was unchanged, backup skipped";
+ return;
+ }
+ }
+
+ $opts->{storage} = $storage->{name};
+ $opts->{maxfiles} = $storage->{maxfiles} if defined($storage->{maxfiles});
+ $opts->{dumpdir} = $storage->{dumpdir};
+ $opts->{tmpdir} = $storage->{dumpdir};
+
+ $opts->{dumpdir} =~ s|/+$|| if ($opts->{dumpdir});
+ $opts->{tmpdir} =~ s|/+$|| if ($opts->{tmpdir});
+
my $tmplog = "$logdir/$vmtype-$vmid.log";
my $lt = localtime();
- my $bkname = "vzdump-$vmtype-$vmid";
my $basename = sprintf "${bkname}-%04d_%02d_%02d-%02d_%02d_%02d",
$lt->year + 1900, $lt->mon + 1, $lt->mday,
$lt->hour, $lt->min, $lt->sec;
@@ -777,9 +842,6 @@ sub exec_backup_task {
$plugin->set_logfd ($logfd);
- # test is VM is running
- my ($running, $status_text) = $plugin->vm_status ($vmid);
-
debugmsg ('info', "status = ${status_text}", $logfd);
# lock VM (prevent config changes)
@@ -1144,8 +1206,14 @@ my $confdesc = {
description => "Use specified hook script.",
optional => 1,
},
- storage => get_standard_option('pve-storage-id', {
- description => "Store resulting file to this storage.",
+ strategy => get_standard_option('strategy', {
+ type => 'string',
+ description => "Backup strategy (when to back up and where to put it).",
+ optional => 1,
+ default => 0,
+ }),
+ storage => get_standard_option('pve-storage-id-list', {
+ description => "Store resulting file to one of these storage (first missing or replace oldest).",
optional => 1,
}),
size => {
@@ -1227,6 +1295,52 @@ sub verify_vzdump_parameters {
}
+sub last_change_entry {
+ my ($vmid) = @_;
+
+ my $filename = "/var/log/pve/tasks/index";
+
+ my $start = undef;
+ my $line;
+
+ my $parse_line = sub {
+ if ($line =~ m/^(\S+)(\s([0-9A-Za-z]{8})(\s(\S.*))?)?$/) {
+ my $upid = $1;
+ my $endtime = $3;
+ my $status = $5;
+ if ((my $task = PVE::Tools::upid_decode($upid, 1))) {
+ return if $task->{type} !~ /stop|shutdown/ || !$task->{id} || $task->{id} ne $vmid;
+
+ $task->{upid} = $upid;
+ $task->{endtime} = hex($endtime) if $endtime;
+ $task->{status} = $status if $status;
+ $start = $task;
+ }
+ }
+ };
+
+ if (my $bw = File::ReadBackwards->new($filename)) {
+ while (defined ($line = $bw->readline)) {
+ &$parse_line();
+ last if defined $start;
+ }
+ $bw->close();
+ }
+ if (!defined $start) {
+ if (my $bw = File::ReadBackwards->new("$filename.1")) {
+ while (defined ($line = $bw->readline)) {
+ &$parse_line();
+ last if defined $start;
+ }
+ $bw->close();
+ }
+ }
+
+ return 0 if !defined $start;
+ return $start->{endtime};
+ return $start;
+}
+
sub command_line {
my ($param) = @_;
@@ -1236,8 +1350,12 @@ sub command_line {
$cmd .= " " . join(' ', PVE::Tools::split_list($param->{vmid}));
}
+ if ($param->{storage} ne '') {
+ $cmd .= " --storage " . join(',', PVE::Tools::split_list($param->{storage}));
+ }
+
foreach my $p (keys %$param) {
- next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' || $p eq 'dow' || $p eq 'stdout';
+ next if $p eq 'id' || $p eq 'vmid' || $p eq 'starttime' || $p eq 'dow' || $p eq 'stdout' || $p eq 'storage';
my $v = $param->{$p};
my $pd = $confdesc->{$p} || die "no such vzdump option '$p'\n";
if ($p eq 'exclude-path') {
diff --git a/www/manager/Makefile b/www/manager/Makefile
index f8cc819..d5958b7 100644
--- a/www/manager/Makefile
+++ b/www/manager/Makefile
@@ -52,6 +52,7 @@ JSSRC= \
form/ContentTypeSelector.js \
form/DayOfWeekSelector.js \
form/BackupModeSelector.js \
+ form/BackupStrategySelector.js \
form/ScsiHwSelector.js \
dc/Tasks.js \
dc/Log.js \
diff --git a/www/manager/dc/Backup.js b/www/manager/dc/Backup.js
index 7241613..72aa942 100644
--- a/www/manager/dc/Backup.js
+++ b/www/manager/dc/Backup.js
@@ -60,6 +60,7 @@ Ext.define('PVE.dc.BackupEdit', {
nodename: 'localhost',
storageContent: 'backup',
allowBlank: false,
+ multiSelect: true,
name: 'storage'
});
@@ -173,7 +174,13 @@ Ext.define('PVE.dc.BackupEdit', {
value: 'snapshot',
name: 'mode'
},
- vmidField
+ {
+ xtype: 'pveBackupStrategySelector',
+ fieldLabel: gettext('Strategy'),
+ value: 'always',
+ name: 'strategy'
+ },
+ vmidField,
];
var ipanel = Ext.create('PVE.panel.InputPanel', {
@@ -279,6 +286,7 @@ Ext.define('PVE.dc.BackupEdit', {
var data = response.result.data;
data.dow = data.dow.split(',');
+ data.storage = data.storage.split(',');
if (data.all || data.exclude) {
if (data.exclude) {
@@ -420,6 +428,12 @@ Ext.define('PVE.dc.BackupView', {
dataIndex: 'storage'
},
{
+ header: gettext('Strategy'),
+ width: 65,
+ sortable: true,
+ dataIndex: 'strategy'
+ },
+ {
header: gettext('Selection'),
flex: 1,
sortable: false,
@@ -460,7 +474,8 @@ Ext.define('PVE.dc.BackupView', {
{ name: 'snapshot', type: 'boolean' },
{ name: 'stop', type: 'boolean' },
{ name: 'suspend', type: 'boolean' },
- { name: 'compress', type: 'boolean' }
+ { name: 'compress', type: 'boolean' },
+ { name: 'strategy', type: 'checkbox' }
]
});
-});
\ No newline at end of file
+});
diff --git a/www/manager/form/BackupStrategySelector.js b/www/manager/form/BackupStrategySelector.js
index e69de29..fe4d10b 100644
--- a/www/manager/form/BackupStrategySelector.js
+++ b/www/manager/form/BackupStrategySelector.js
@@ -0,0 +1,17 @@
+Ext.define('PVE.form.BackupStrategySelector', {
+ extend: 'PVE.form.KVComboBox',
+ alias: ['widget.pveBackupStrategySelector'],
+
+ initComponent: function() {
+ var me = this;
+
+ me.data = [
+ ['always', gettext('Always backup VMs - replace oldest backup')],
+ ['distribute', gettext('Always backup VMs - replace oldest backup on storage with least up to date backup')],
+ ['aggressive', gettext('Skip backup whenever theere is at least one unchanged backup')],
+ ['safe', gettext('Skip backup when all storages have at least one unchanged backup')]
+ ];
+
+ me.callParent();
+ }
+});
diff --git a/www/manager/grid/BackupView.js b/www/manager/grid/BackupView.js
index 0ebad8f..dabace6 100644
--- a/www/manager/grid/BackupView.js
+++ b/www/manager/grid/BackupView.js
@@ -68,6 +68,7 @@ Ext.define('PVE.grid.BackupView', {
labelAlign: 'right',
storageContent: 'backup',
allowBlank: false,
+ multiSelect: false,
listeners: {
change: function(f, value) {
setStorage(value);
diff --git a/www/manager/window/Backup.js b/www/manager/window/Backup.js
index f7b30d5..ccbbdb6 100644
--- a/www/manager/window/Backup.js
+++ b/www/manager/window/Backup.js
@@ -22,6 +22,7 @@ Ext.define('PVE.window.Backup', {
nodename: me.nodename,
name: 'storage',
value: me.storage,
+ multiSelect: true,
fieldLabel: gettext('Storage'),
storageContent: 'backup',
allowBlank: false
@@ -43,6 +44,12 @@ Ext.define('PVE.window.Backup', {
name: 'mode'
},
{
+ xtype: 'pveBackupStrategySelector',
+ fieldLabel: gettext('Strategy'),
+ value: 'always',
+ name: 'strategy'
+ },
+ {
xtype: 'pveCompressionSelector',
name: 'compress',
value: 'lzo',
--
1.7.10.4
More information about the pve-devel
mailing list