[pve-devel] [PATCH storage v3 7/7] Fix: #2124 storage: add zstd support
Alwin Antreich
a.antreich at proxmox.com
Wed Apr 8 12:26:02 CEST 2020
Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
PVE/Storage.pm | 4 +++-
PVE/Storage/Plugin.pm | 2 +-
test/test_archive_info.pm | 9 ++++++---
test/test_list_volumes.pm | 4 ++++
test/test_parse_volname.pm | 3 +++
test/test_path_to_volume_id.pm | 2 ++
6 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 46384ff..df477a7 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1295,10 +1295,12 @@ sub decompressor_info {
tar => {
gz => ['tar', '-z'],
lzo => ['tar', '--lzop'],
+ zst => ['tar', '--zstd'],
},
vma => {
gz => ['zcat'],
lzo => ['lzop', '-d', '-c'],
+ zst => ['zstd', '-q', '-d', '-c'],
},
};
@@ -1389,7 +1391,7 @@ sub extract_vzdump_config_vma {
my $errstring;
my $err = sub {
my $output = shift;
- if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/) {
+ if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error : Broken pipe/) {
$broken_pipe = 1;
} elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
$errstring = "Failed to extract config from VMA archive: $output\n";
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index ef6e6de..ec434a4 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -17,7 +17,7 @@ use JSON;
use base qw(PVE::SectionConfig);
-use constant COMPRESSOR_RE => 'gz|lzo';
+use constant COMPRESSOR_RE => 'gz|lzo|zst';
our @COMMON_TAR_FLAGS = qw(
--one-file-system
diff --git a/test/test_archive_info.pm b/test/test_archive_info.pm
index 464cc89..cd126f8 100644
--- a/test/test_archive_info.pm
+++ b/test/test_archive_info.pm
@@ -10,19 +10,22 @@ use Test::More;
my @tests = (
# backup archives
- [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => undef, 'compression' => undef }, 'Backup archive, vma' ],
- [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['zcat'], 'compression' => 'gz' }, 'Backup archive, vma, gz' ],
- [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['lzop', '-d', '-c'], 'compression' => 'lzo' }, 'Backup archive, vma, lzo' ],
+ [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => undef, 'compression' => undef }, 'Backup archive, vma' ],
+ [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['zcat'], 'compression' => 'gz' }, 'Backup archive, vma, gz' ],
+ [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['lzop', '-d', '-c'], 'compression' => 'lzo' }, 'Backup archive, vma, lzo' ],
+ [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.zst', { 'type' => 'qemu', 'format' => 'vma', 'decompressor' => ['zstd', '-q', '-d', '-c'], 'compression' => 'zst' }, 'Backup archive, vma, zst' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => undef, 'compression' => undef }, 'Backup archive, lxc' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.gz', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' }, 'Backup archive, lxc, gz' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tgz', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' }, 'Backup archive, lxc, tgz' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '--lzop'], 'compression' => 'lzo' }, 'Backup archive, lxc, lzo' ],
+ [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst', { 'type' => 'lxc', 'format' => 'tar', 'decompressor' => ['tar', '--zstd'], 'compression' => 'zst' }, 'Backup archive, lxc, zst' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => undef, 'compression' => undef }, 'Backup archive, openvz' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.gz', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' }, 'Backup archive, openvz, gz' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tgz', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '-z'], 'compression' => 'gz' }, 'Backup archive, openvz, tgz' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.lzo', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '--lzop'], 'compression' => 'lzo' }, 'Backup archive, openvz, lzo' ],
+ [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.zst', { 'type' => 'openvz', 'format' => 'tar', 'decompressor' => ['tar', '--zstd'], 'compression' => 'zst' }, 'Backup archive, openvz, zst' ],
# failed matches
[ 'backup/vzdump-none-16112-2020_03_30-21_39_30.tar', "ERROR: couldn't determine format and compression type\n", 'Failed match: Backup archive, no virtualization type' ],
diff --git a/test/test_list_volumes.pm b/test/test_list_volumes.pm
index 169c8be..5a27d81 100644
--- a/test/test_list_volumes.pm
+++ b/test/test_list_volumes.pm
@@ -99,6 +99,7 @@ my @tests = (
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
+ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
"$storage_dir/snippets/userconfig.yaml",
"$storage_dir/snippets/hookscript.pl",
],
@@ -109,6 +110,7 @@ my @tests = (
{ 'content' => 'backup', 'ctime' => 1585595500, 'format' => 'vma.gz', 'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz' },
{ 'content' => 'backup', 'ctime' => 1585595565, 'format' => 'vma.lzo', 'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo' },
{ 'content' => 'backup', 'ctime' => 1585595635, 'format' => 'vma', 'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma' },
+ { 'content' => 'backup', 'ctime' => 1585595635, 'format' => 'vma.zst', 'size' => DEFAULT_SIZE, 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst' },
{ 'content' => 'snippets', 'ctime' => DEFAULT_CTIME, 'format' => 'snippet', 'size' => DEFAULT_SIZE, 'volid' => 'local:snippets/hookscript.pl' },
{ 'content' => 'snippets', 'ctime' => DEFAULT_CTIME, 'format' => 'snippet', 'size' => DEFAULT_SIZE, 'volid' => 'local:snippets/userconfig.yaml' },
],
@@ -120,12 +122,14 @@ my @tests = (
"$storage_dir/images/16112/vm-16112-disk-0.raw",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
+ "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
],
expected => [
{ 'content' => 'rootdir', 'ctime' => DEFAULT_CTIME, 'format' => 'raw', 'parent' => undef, 'size' => DEFAULT_SIZE, 'used' => DEFAULT_USED, 'vmid' => '16112', 'volid' => 'local:16112/vm-16112-disk-0.raw' },
{ 'content' => 'backup', 'ctime' => 1585597170, 'format' => 'tar.lzo', 'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo' },
{ 'content' => 'backup', 'ctime' => 1585597770, 'format' => 'tar.gz', 'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz' },
+ { 'content' => 'backup', 'ctime' => 1585597770, 'format' => 'tar.zst', 'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst' },
{ 'content' => 'backup', 'ctime' => 1585598370, 'format' => 'tgz', 'size' => DEFAULT_SIZE, 'vmid' => '16112', 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz' },
],
description => 'VMID: 16112, lxc, raw, backup',
diff --git a/test/test_parse_volname.pm b/test/test_parse_volname.pm
index 84665d3..27f2249 100644
--- a/test/test_parse_volname.pm
+++ b/test/test_parse_volname.pm
@@ -40,16 +40,19 @@ my @tests = (
[ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma', ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma', '16110'], 'Backup archive, vma' ],
[ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz', ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma.gz', '16110'], 'Backup archive, vma, gz' ],
[ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma.lzo', '16110'], 'Backup archive, vma, lzo' ],
+ [ 'backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.zst', ['backup', 'vzdump-qemu-16110-2020_03_30-21_12_40.vma.zst', '16110'], 'Backup archive, vma, zst' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar', '16112'], 'Backup archive, lxc' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.gz', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar.gz', '16112'], 'Backup archive, lxc, gz' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tgz', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tgz', '16112'], 'Backup archive, lxc, tgz' ],
[ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', '16112'], 'Backup archive, lxc, lzo' ],
+ [ 'backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst', ['backup', 'vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst', '16112'], 'Backup archive, lxc, zst' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar', '16112'], 'Backup archive, openvz' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.gz', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar.gz', '16112'], 'Backup archive, openvz, gz' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tgz', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tgz', '16112'], 'Backup archive, openvz, tgz' ],
[ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.lzo', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar.lzo', '16112'], 'Backup archive, openvz, lzo' ],
+ [ 'backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.zst', ['backup', 'vzdump-openvz-16112-2020_03_30-21_39_30.tar.zst', '16112'], 'Backup archive, openvz, zst' ],
[ 'backup/vzdump-none-16112-2020_03_30-21_39_30.tar', ['backup', 'vzdump-none-16112-2020_03_30-21_39_30.tar'], 'Backup archive, no virtualization type' ],
diff --git a/test/test_path_to_volume_id.pm b/test/test_path_to_volume_id.pm
index e693974..83caf24 100644
--- a/test/test_path_to_volume_id.pm
+++ b/test/test_path_to_volume_id.pm
@@ -44,8 +44,10 @@ my @tests = (
[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz", ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz'], 'Backup, vma.gz' ],
[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo", ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo'], 'Backup, vma.lzo' ],
+ [ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst", ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst'], 'Backup, vma.zst' ],
[ "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma", ['iso', 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma'], 'Backup, vma' ],
[ "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo", ['iso', 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo'], 'Backup, tar.lzo' ],
+ [ "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst", ['iso', 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst'], 'Backup, tar.zst' ],
[ "$storage_dir/template/iso/yet-again-a-installation-disk.iso", ['iso', 'local:iso/yet-again-a-installation-disk.iso'], 'ISO file' ],
[ "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz", ['vztmpl', 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz'], 'CT template, tar.gz' ],
--
2.20.1
More information about the pve-devel
mailing list