[pve-devel] [PATCH storage 1/3] Fix #2124: Add support for zstd
Alwin Antreich
a.antreich at proxmox.com
Fri Jun 14 15:37:30 CEST 2019
This patch adds zstd for backup/restore. It also factors out the common
parts on the decompression tools. Sadly tar 1.31 (includes zstd) was not
available at the time of writing this patch.
Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
PVE/Storage.pm | 124 ++++++++++++++++++++++++++++++++++++++++----------
PVE/Storage/Plugin.pm | 2 +-
2 files changed, 102 insertions(+), 24 deletions(-)
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 588e775..f3c50ca 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -510,7 +510,7 @@ sub path_to_volume_id {
} elsif ($path =~ m!^$privatedir/(\d+)$!) {
my $vmid = $1;
return ('rootdir', "$sid:rootdir/$vmid");
- } elsif ($path =~ m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) {
+ } elsif ($path =~ m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!) {
my $name = $1;
return ('iso', "$sid:backup/$name");
}
@@ -862,7 +862,7 @@ sub template_list {
$info = { volid => "$sid:vztmpl/$1", format => "t$2" };
} elsif ($tt eq 'backup') {
- next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!;
+ next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!;
$info = { volid => "$sid:backup/$1", format => $2 };
}
@@ -1362,36 +1362,112 @@ sub foreach_volid {
}
}
-sub extract_vzdump_config_tar {
- my ($archive, $conf_re) = @_;
-
- die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
-
- my $pid = open(my $fh, '-|', 'tar', 'tf', $archive) ||
- die "unable to open file '$archive'\n";
+sub decompressor_info {
+ my ($archive, $comp, $noerr) = @_;
+ my $format;
+ if (!defined($comp)) {
+ my $volid = basename($archive);
+ if ($volid =~ /vzdump-(lxc|openvz|qemu)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/) {
+ if ($8 eq 'tgz') {
+ $format = 'tar';
+ $comp = 'gz';
+ } else {
+ $format = $10;
+ $comp = $12 if defined($12);
+ }
+ } else {
+ die "ERROR: couldn't determine format and compression type\n" if !$noerr;
+ }
+ }
- my $file;
- while (defined($file = <$fh>)) {
- if ($file =~ $conf_re) {
- $file = $1; # untaint
- last;
+ my $cmd;
+ if (defined($comp)) {
+ if ($comp eq 'gz') {
+ $cmd = ["zcat", $archive];
+ } elsif ($comp eq 'lzo') {
+ $cmd = ["lzop", "-d", "-c", $archive];
+ } elsif ($comp eq 'zst') {
+ $cmd = ["zstd", "-q", "-d", "-c", $archive];
+ } else {
+ die "unknown compression method '$comp'\n" if !$noerr;
}
+ } else {
+ die "compression type not set\n" if !$noerr;
}
- kill 15, $pid;
- waitpid $pid, 0;
- close $fh;
+ pop(@$cmd) if !defined($archive);
- die "ERROR: archive contains no configuration file\n" if !$file;
- chomp $file;
+ return $cmd;
+}
+sub extract_from_archive {
+ my ($cmd) = @_;
my $raw = '';
my $out = sub {
my $output = shift;
$raw .= "$output\n";
};
+ # in some cases, lzop|zcat|zstd exits with 1 when its stdout pipe is closed
+ # early, detect this and ignore the exit code later
+ my $broken_pipe;
+ my $errstring;
+ my $err = sub {
+ my $output = shift;
+ if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/Error 70 : Write error :/i) {
+ $broken_pipe = 1;
+ } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
+ $errstring = "Failed to extract config from archive: $output\n";
+ }
+ };
+
+ # in other cases, the pipeline will exit with exit code 141
+ # because of the broken pipe, handle / ignore this as well
+ my $rc;
+ eval {
+ $rc = PVE::Tools::run_command($cmd, outfunc => $out, errfunc => $err, noerr => 1);
+ };
+ my $rerr = $@;
+
+ # use exit code if no stderr output and not just broken pipe
+ if (!$errstring && !$broken_pipe && $rc != 0 && $rc != 141) {
+ die "$rerr\n" if $rerr;
+ die "config extraction failed with exit code $rc\n";
+ }
+ die "$errstring\n" if $errstring;
+
+ return $raw;
+
+}
+sub extract_vzdump_config_tar {
+ my ($archive, $conf_re) = @_;
+
+ die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
+
+ my $decomp = decompressor_info($archive, undef, 1);
- PVE::Tools::run_command(['tar', '-xpOf', $archive, $file, '--occurrence'], outfunc => $out);
+ my $file;
+ my $file_list = sub {
+ my (@flst) = shift;
+
+ foreach my $line (@flst) {
+ if ($line =~ $conf_re) {
+ $file = $1; # untaint
+ last;
+ }
+ }
+
+ die "ERROR: archive contains no configuration file\n" if !$file;
+ };
+
+ my $raw;
+
+ if ($decomp) {
+ PVE::Tools::run_command([$decomp, ['tar', '-tf', '-']], outfunc => $file_list);
+ $raw = extract_from_archive([$decomp, ['tar', '-xpO', "$file", '--occurrence']]);
+ } else {
+ PVE::Tools::run_command(['tar', '-tf', "$archive"], outfunc => $file_list);
+ $raw = extract_from_archive(['tar', '-xpO', "$file", '--occurrence', '-f', "$archive"]);
+ }
return wantarray ? ($raw, $file) : $raw;
}
@@ -1413,6 +1489,8 @@ sub extract_vzdump_config_vma {
$uncomp = ["zcat", $archive];
} elsif ($comp eq 'lzo') {
$uncomp = ["lzop", "-d", "-c", $archive];
+ } elsif ($comp eq 'zst') {
+ $uncomp = ["zstd", "-q", "-d", "-c", $archive];
} else {
die "unknown compression method '$comp'\n";
}
@@ -1424,7 +1502,7 @@ sub extract_vzdump_config_vma {
my $errstring;
my $err = sub {
my $output = shift;
- if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/) {
+ if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/Error 70 : Write error :/) {
$broken_pipe = 1;
} elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
$errstring = "Failed to extract config from VMA archive: $output\n";
@@ -1458,9 +1536,9 @@ sub extract_vzdump_config {
my $archive = abs_filesystem_path($cfg, $volid);
- if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/) {
+ if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo|zst))?))$/) {
return extract_vzdump_config_tar($archive, qr!^(\./etc/vzdump/(pct|vps)\.conf)$!);
- } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/) {
+ } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/) {
my $format;
my $comp;
if ($7 eq 'tgz') {
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 255c643..cdb9bd8 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -421,7 +421,7 @@ sub parse_volname {
return ('vztmpl', $1);
} elsif ($volname =~ m!^rootdir/(\d+)$!) {
return ('rootdir', $1, $1);
- } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) {
+ } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst)))$!) {
my $fn = $1;
if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) {
return ('backup', $fn, $2);
--
2.11.0
More information about the pve-devel
mailing list