[pve-devel] [PATCH storage 1/3] Fix #2124: Add support for zstd

Alwin Antreich a.antreich at proxmox.com
Fri Jun 14 15:37:30 CEST 2019


This patch adds zstd for backup/restore. It also factors out the common
parts on the decompression tools. Sadly tar 1.31 (includes zstd) was not
available at the time of writing this patch.

Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
 PVE/Storage.pm        | 124 ++++++++++++++++++++++++++++++++++++++++----------
 PVE/Storage/Plugin.pm |   2 +-
 2 files changed, 102 insertions(+), 24 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 588e775..f3c50ca 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -510,7 +510,7 @@ sub path_to_volume_id {
 	} elsif ($path =~ m!^$privatedir/(\d+)$!) {
 	    my $vmid = $1;
 	    return ('rootdir', "$sid:rootdir/$vmid");
-	} elsif ($path =~ m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) {
+	} elsif ($path =~ m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!) {
 	    my $name = $1;
 	    return ('iso', "$sid:backup/$name");
 	}
@@ -862,7 +862,7 @@ sub template_list {
 		    $info = { volid => "$sid:vztmpl/$1", format => "t$2" };
 
 		} elsif ($tt eq 'backup') {
-		    next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!;
+		    next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!;
 
 		    $info = { volid => "$sid:backup/$1", format => $2 };
 		}
@@ -1362,36 +1362,112 @@ sub foreach_volid {
     }
 }
 
-sub extract_vzdump_config_tar {
-    my ($archive, $conf_re) = @_;
-
-    die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
-
-    my $pid = open(my $fh, '-|', 'tar', 'tf', $archive) ||
-       die "unable to open file '$archive'\n";
+sub decompressor_info {
+    my ($archive, $comp, $noerr) = @_;
+    my $format;
+    if (!defined($comp)) {
+	my $volid = basename($archive);
+	if ($volid =~ /vzdump-(lxc|openvz|qemu)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/) {
+	    if ($8 eq 'tgz') {
+		$format = 'tar';
+		$comp = 'gz';
+	    } else {
+		$format = $10;
+		$comp = $12 if defined($12);
+	    }
+	} else {
+	    die "ERROR: couldn't determine format and compression type\n" if !$noerr;
+	}
+    }
 
-    my $file;
-    while (defined($file = <$fh>)) {
-	if ($file =~ $conf_re) {
-	    $file = $1; # untaint
-	    last;
+    my $cmd;
+    if (defined($comp)) {
+	if ($comp eq 'gz') {
+	    $cmd = ["zcat", $archive];
+	} elsif ($comp eq 'lzo') {
+	    $cmd = ["lzop", "-d", "-c", $archive];
+	} elsif ($comp eq 'zst') {
+	    $cmd = ["zstd", "-q", "-d", "-c", $archive];
+	} else {
+	    die "unknown compression method '$comp'\n" if !$noerr;
 	}
+    } else {
+	die "compression type not set\n" if !$noerr;
     }
 
-    kill 15, $pid;
-    waitpid $pid, 0;
-    close $fh;
+    pop(@$cmd) if !defined($archive);
 
-    die "ERROR: archive contains no configuration file\n" if !$file;
-    chomp $file;
+    return $cmd;
+}
 
+sub extract_from_archive {
+    my ($cmd) = @_;
     my $raw = '';
     my $out = sub {
 	my $output = shift;
 	$raw .= "$output\n";
     };
+    # in some cases, lzop|zcat|zstd exits with 1 when its stdout pipe is closed
+    # early, detect this and ignore the exit code later
+    my $broken_pipe;
+    my $errstring;
+    my $err = sub {
+	my $output = shift;
+	if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/Error 70 : Write error :/i) {
+	$broken_pipe = 1;
+	} elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
+	$errstring = "Failed to extract config from archive: $output\n";
+	}
+    };
+
+    # in other cases, the pipeline will exit with exit code 141
+    # because of the broken pipe, handle / ignore this as well
+    my $rc;
+    eval {
+	$rc = PVE::Tools::run_command($cmd, outfunc => $out, errfunc => $err, noerr => 1);
+    };
+    my $rerr = $@;
+
+    # use exit code if no stderr output and not just broken pipe
+    if (!$errstring && !$broken_pipe && $rc != 0 && $rc != 141) {
+	die "$rerr\n" if $rerr;
+	die "config extraction failed with exit code $rc\n";
+    }
+    die "$errstring\n" if $errstring;
+
+    return $raw;
+
+}
+sub extract_vzdump_config_tar {
+    my ($archive, $conf_re) = @_;
+
+    die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
+
+    my $decomp = decompressor_info($archive, undef, 1);
 
-    PVE::Tools::run_command(['tar', '-xpOf', $archive, $file, '--occurrence'], outfunc => $out);
+    my $file;
+    my $file_list = sub {
+	my (@flst) = shift;
+
+	foreach my $line (@flst) {
+	    if ($line =~ $conf_re) {
+		$file = $1; # untaint
+		last;
+	    }
+	}
+
+	die "ERROR: archive contains no configuration file\n" if !$file;
+    };
+
+    my $raw;
+
+    if ($decomp) {
+	PVE::Tools::run_command([$decomp, ['tar', '-tf', '-']], outfunc => $file_list);
+	$raw = extract_from_archive([$decomp, ['tar', '-xpO', "$file", '--occurrence']]);
+    } else {
+	PVE::Tools::run_command(['tar', '-tf', "$archive"], outfunc => $file_list);
+	$raw = extract_from_archive(['tar', '-xpO', "$file", '--occurrence', '-f', "$archive"]);
+    }
 
     return wantarray ? ($raw, $file) : $raw;
 }
@@ -1413,6 +1489,8 @@ sub extract_vzdump_config_vma {
 	    $uncomp = ["zcat", $archive];
 	} elsif ($comp eq 'lzo') {
 	    $uncomp = ["lzop", "-d", "-c", $archive];
+	} elsif ($comp eq 'zst') {
+	    $uncomp = ["zstd", "-q", "-d", "-c", $archive];
 	} else {
 	    die "unknown compression method '$comp'\n";
 	}
@@ -1424,7 +1502,7 @@ sub extract_vzdump_config_vma {
 	my $errstring;
 	my $err = sub {
 	    my $output = shift;
-	    if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/) {
+	    if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/Error 70 : Write error :/) {
 		$broken_pipe = 1;
 	    } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
 		$errstring = "Failed to extract config from VMA archive: $output\n";
@@ -1458,9 +1536,9 @@ sub extract_vzdump_config {
 
     my $archive = abs_filesystem_path($cfg, $volid);
 
-    if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/) {
+    if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo|zst))?))$/) {
 	return extract_vzdump_config_tar($archive, qr!^(\./etc/vzdump/(pct|vps)\.conf)$!);
-    } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/) {
+    } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/) {
 	my $format;
 	my $comp;
 	if ($7 eq 'tgz') {
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 255c643..cdb9bd8 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -421,7 +421,7 @@ sub parse_volname {
 	return ('vztmpl', $1);
     } elsif ($volname =~ m!^rootdir/(\d+)$!) {
 	return ('rootdir', $1, $1);
-    } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) {
+    } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst)))$!) {
 	my $fn = $1;
 	if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) {
 	    return ('backup', $fn, $2);
-- 
2.11.0





More information about the pve-devel mailing list