[pve-devel] [PATCH storage v3 7/7] Fix: #2124 storage: add zstd support

Alwin Antreich a.antreich at proxmox.com
Wed Apr 8 12:26:02 CEST 2020


Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
 PVE/Storage.pm                 | 4 +++-
 PVE/Storage/Plugin.pm          | 2 +-
 test/test_archive_info.pm      | 9 ++++++---
 test/test_list_volumes.pm      | 4 ++++
 test/test_parse_volname.pm     | 3 +++
 test/test_path_to_volume_id.pm | 2 ++
 6 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 46384ff..df477a7 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1295,10 +1295,12 @@ sub decompressor_info {
 	tar => {
 	    gz => ['tar', '-z'],
 	    lzo => ['tar', '--lzop'],
+	    zst => ['tar', '--zstd'],
 	},
 	vma => {
 	    gz => ['zcat'],
 	    lzo => ['lzop', '-d', '-c'],
+	    zst => ['zstd', '-q', '-d', '-c'],
 	},
     };
 
@@ -1389,7 +1391,7 @@ sub extract_vzdump_config_vma {
 	my $errstring;
 	my $err = sub {
 	    my $output = shift;
-	    if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/) {
+	    if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error : Broken pipe/) {
 		$broken_pipe = 1;
 	    } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
 		$errstring = "Failed to extract config from VMA archive: $output\n";
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index ef6e6de..ec434a4 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -17,7 +17,7 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
-use constant COMPRESSOR_RE => 'gz|lzo';
+use constant COMPRESSOR_RE => 'gz|lzo|zst';
 
 our @COMMON_TAR_FLAGS = qw(
     --one-file-system
diff --git a/test/test_archive_info.pm b/test/test_archive_info.pm
index 464cc89..cd126f8 100644
--- a/test/test_archive_info.pm
+++ b/test/test_archive_info.pm
@@ -10,19 +10,22 @@ use Test::More;
 
 my @tests = (
     # backup archives
-    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma',     { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => undef, 'compression' => undef },                'Backup archive, vma' ],
-    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz',  { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['zcat'], 'compression' => 'gz' },              'Backup archive, vma, gz' ],
-    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['lzop', '-d', '-c'], 'compression' => 'lzo' }, 'Backup archive, vma, lzo' ],
+    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma',     { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => undef, 'compression' => undef },                      'Backup archive, vma' ],
+    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz',  { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['zcat'], 'compression' => 'gz' },                    'Backup archive, vma, gz' ],
+    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['lzop', '-d', '-c'], 'compression' => 'lzo' },       'Backup archive, vma, lzo' ],
+    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.zst', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['zstd', '-q', '-d', '-c'], 'compression' => 'zst' }, 'Backup archive, vma, zst' ],
 
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar',     { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => undef, 'compression' => undef },             'Backup archive, lxc' ],
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.gz',  { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' },      'Backup archive, lxc, gz' ],
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tgz',     { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' },      'Backup archive, lxc, tgz' ],
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '--lzop'], 'compression' => 'lzo' }, 'Backup archive, lxc, lzo' ],
+    [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '--zstd'], 'compression' => 'zst' }, 'Backup archive, lxc, zst' ],
 
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar',     { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => undef, 'compression' => undef },             'Backup archive, openvz' ],
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.gz',  { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' },      'Backup archive, openvz, gz' ],
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tgz',     { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' },      'Backup archive, openvz, tgz' ],
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.lzo', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '--lzop'], 'compression' => 'lzo' }, 'Backup archive, openvz, lzo' ],
+    [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.zst', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '--zstd'], 'compression' => 'zst' }, 'Backup archive, openvz, zst' ],
 
     # failed matches
     [ 'backup/vzdump-none-16112-2020_03_30-21_39_30.tar',       "ERROR: couldn't determine format and compression type\n", 'Failed match: Backup archive, no virtualization type' ],
diff --git a/test/test_list_volumes.pm b/test/test_list_volumes.pm
index 169c8be..5a27d81 100644
--- a/test/test_list_volumes.pm
+++ b/test/test_list_volumes.pm
@@ -99,6 +99,7 @@ my @tests = (
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
+	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
 	    "$storage_dir/snippets/userconfig.yaml",
 	    "$storage_dir/snippets/hookscript.pl",
 	],
@@ -109,6 +110,7 @@ my @tests = (
 	    { 'content' => 'backup',   'ctime' => 1585595500,    'format' => 'vma.gz',  'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz' },
 	    { 'content' => 'backup',   'ctime' => 1585595565,    'format' => 'vma.lzo', 'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo' },
 	    { 'content' => 'backup',   'ctime' => 1585595635,    'format' => 'vma',     'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma' },
+	    { 'content' => 'backup',   'ctime' => 1585595635,    'format' => 'vma.zst',     'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst' },
 	    { 'content' => 'snippets', 'ctime' => DEFAULT_CTIME, 'format' => 'snippet', 'size' => DEFAULT_SIZE, 'volid' => 'local:snippets/hookscript.pl' },
 	    { 'content' => 'snippets', 'ctime' => DEFAULT_CTIME, 'format' => 'snippet', 'size' => DEFAULT_SIZE, 'volid' => 'local:snippets/userconfig.yaml' },
 	],
@@ -120,12 +122,14 @@ my @tests = (
 	    "$storage_dir/images/16112/vm-16112-disk-0.raw",
 	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
 	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
+	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst",
 	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
 	],
 	expected => [
 	    { 'content' => 'rootdir',  'ctime' => DEFAULT_CTIME, 'format' => 'raw', 'parent' => undef, 'size' => DEFAULT_SIZE, 'used' => DEFAULT_USED, 'vmid' => '16112', 'volid' => 'local:16112/vm-16112-disk-0.raw' },
 	    { 'content' => 'backup',   'ctime' => 1585597170,    'format' => 'tar.lzo', 'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo' },
 	    { 'content' => 'backup',   'ctime' => 1585597770,    'format' => 'tar.gz',  'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz' },
+	    { 'content' => 'backup',   'ctime' => 1585597770,    'format' => 'tar.zst',  'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst' },
 	    { 'content' => 'backup',   'ctime' => 1585598370,    'format' => 'tgz',     'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz' },
 	],
 	description => 'VMID: 16112, lxc, raw, backup',
diff --git a/test/test_parse_volname.pm b/test/test_parse_volname.pm
index 84665d3..27f2249 100644
--- a/test/test_parse_volname.pm
+++ b/test/test_parse_volname.pm
@@ -40,16 +40,19 @@ my @tests = (
     [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma',     ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma',     '16110'], 'Backup archive, vma' ],
     [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz',  ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz',  '16110'], 'Backup archive, vma, gz' ],
     [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', '16110'], 'Backup archive, vma, lzo' ],
+    [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.zst', ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma.zst', '16110'], 'Backup archive, vma, zst' ],
 
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar',     ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar',     '16112'], 'Backup archive, lxc' ],
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.gz',  ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar.gz',  '16112'], 'Backup archive, lxc, gz' ],
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tgz',     ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tgz',     '16112'], 'Backup archive, lxc, tgz' ],
     [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', '16112'], 'Backup archive, lxc, lzo' ],
+    [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst', '16112'], 'Backup archive, lxc, zst' ],
 
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar',     ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar',     '16112'], 'Backup archive, openvz' ],
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.gz',  ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar.gz',  '16112'], 'Backup archive, openvz, gz' ],
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tgz',     ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tgz',     '16112'], 'Backup archive, openvz, tgz' ],
     [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.lzo', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar.lzo', '16112'], 'Backup archive, openvz, lzo' ],
+    [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.zst', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar.zst', '16112'], 'Backup archive, openvz, zst' ],
 
     [ 'backup/vzdump-none-16112-2020_03_30-21_39_30.tar',       ['backup', 'vzdump-none-16112-2020_03_30-21_39_30.tar'],       'Backup archive, no virtualization type' ],
 
diff --git a/test/test_path_to_volume_id.pm b/test/test_path_to_volume_id.pm
index e693974..83caf24 100644
--- a/test/test_path_to_volume_id.pm
+++ b/test/test_path_to_volume_id.pm
@@ -44,8 +44,10 @@ my @tests = (
 
 	[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",  ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz'],  'Backup, vma.gz' ],
 	[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo", ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo'], 'Backup, vma.lzo' ],
+	[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst", ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst'], 'Backup, vma.zst' ],
 	[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",     ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma'],     'Backup, vma' ],
 	[ "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",  ['iso', 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo'],  'Backup, tar.lzo' ],
+	[ "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst",  ['iso', 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst'],  'Backup, tar.zst' ],
 
 	[ "$storage_dir/template/iso/yet-again-a-installation-disk.iso",          ['iso', 'local:iso/yet-again-a-installation-disk.iso'],              'ISO file' ],
 	[ "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz", ['vztmpl', 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz'], 'CT template, tar.gz' ],
-- 
2.20.1





More information about the pve-devel mailing list