[pve-devel] Using zstd for backup
Nils Privat
gismo3303 at gmail.com
Fri Feb 16 10:09:58 CET 2018
Nice patch, thanks. The patch works straightforward, we just have to
include "zstd: 1" in "/etc/vzdump.conf" to be able to use more CPUs.
In order to use this patch we have to clone and make zstd first because the
delivered zstd on debian stretch is to old.
Question @PVE-Devel: To finish this patch up we have to include a clone of
https://github.com/facebook/zstd to your git repositories
https://git.proxmox.com, is that possible?
Thanks
2017-11-06 12:49 GMT+01:00 Oliver Jaksch <proxmox-devel at com-in.de>:
> Some users, including me, are discussing about the possibilities to extend
> the backup routines to offer more compression programs:
> https://forum.proxmox.com/threads/suggestion-add-lrzip-
> to-backup-compression-options.29496
>
> I've made a small set of patches to be able to use zstd (
> https://github.com/facebook/zstd/).
> Debian/stretch offers zstd, but it is too old; I used zstd 1.3.x to gain
> access for controlling the use of threads like using pigz.
>
> The patches applies cleanly at my pve 5.1 (and another fresh system) and
> are working fine to backup/restore lxc containers and qemu VMs.
> So please have a look at these patches to get it maybe official sometimes.
> Thanks, over and out
> -
> Oliver
>
>
> --- /usr/share/pve-manager.orig/js/pvemanagerlib.js 2017-10-27
> 12:25:26.000000000 +0200
> +++ /usr/share/pve-manager/js/pvemanagerlib.js 2017-11-04
> 17:30:37.626121574 +0100
> @@ -6044,7 +6044,8 @@
> comboItems: [
> ['0', PVE.Utils.noneText],
> ['lzo', 'LZO (' + gettext('fast') + ')'],
> - ['gzip', 'GZIP (' + gettext('good') + ')']
> + ['gzip', 'GZIP (' + gettext('good') + ')'],
> + ['zstd', 'ZSTD (' + gettext('best') + ')']
> ]
> });
> Ext.define('PVE.form.PoolSelector', {
> --- /usr/share/perl5/PVE.org/QemuServer.pm 2017-11-04
> 17:00:48.000000000 +0100
> +++ /usr/share/perl5/PVE/QemuServer.pm 2017-11-05 16:37:08.438200797
> +0100
> @@ -5222,6 +5222,9 @@
> } elsif ($archive =~ m/.tar.lzo$/) {
> $format = 'tar' if !$format;
> $comp = 'lzop';
> + } elsif ($archive =~ m/.tar.zst$/) {
> + $format = 'tar' if !$format;
> + $comp = 'zstd';
> } elsif ($archive =~ m/\.vma$/) {
> $format = 'vma' if !$format;
> } elsif ($archive =~ m/\.vma\.gz$/) {
> @@ -5230,6 +5233,9 @@
> } elsif ($archive =~ m/\.vma\.lzo$/) {
> $format = 'vma' if !$format;
> $comp = 'lzop';
> + } elsif ($archive =~ m/\.vma\.zst$/) {
> + $format = 'vma' if !$format;
> + $comp = 'zstd';
> } else {
> $format = 'vma' if !$format; # default
> }
> @@ -5472,6 +5478,8 @@
> $uncomp = "zcat $qarchive|";
> } elsif ($comp eq 'lzop') {
> $uncomp = "lzop -d -c $qarchive|";
> + } elsif ($comp eq 'zstd') {
> + $uncomp = "zstd -d -c -q $qarchive|";
> } else {
> die "unknown compression method '$comp'\n";
> }
> @@ -5746,10 +5754,17 @@
> $tocmd .= ' --prealloc' if $opts->{prealloc};
> $tocmd .= ' --info' if $opts->{info};
>
> - # tar option "xf" does not autodetect compression when read from
> STDIN,
> - # so we pipe to zcat
> - my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " "
> .
> - PVE::Tools::shellquote("--to-command=$tocmd");
> + my $cmd;
> + if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/ || $archive =~
> m/.tar.lzo$/) {
> + # tar option "xf" does not autodetect compression when read from
> STDIN,
> + # so we pipe to zcat
> + $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " "
> .
> + PVE::Tools::shellquote("--to-command=$tocmd");
> + } elsif ($archive =~ m/.tar.zst$/) {
> + $cmd = "zcat -f|tar -I zstd -xf " . PVE::Tools::shellquote($archive)
> . " " .
> + PVE::Tools::shellquote("--to-command=$tocmd");
> + }
> + warn "#################### $cmd\n";
>
> my $tmpdir = "/var/tmp/vzdumptmp$$";
> mkpath $tmpdir;
> @@ -6483,7 +6498,7 @@
> my $res = [];
> foreach my $id (keys %$data) {
> foreach my $item (@{$data->{$id}}) {
> - next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
> + next if $item->{format} !~ m/^vma\.(gz|lzo|zst)$/;
> push @$res, $item->{volid} if defined($item->{volid});
> }
> }
> --- /usr/share/perl5/PVE.org/Storage.pm 2017-10-17 15:02:17.000000000
> +0200
> +++ /usr/share/perl5/PVE/Storage.pm 2017-11-05 16:25:18.312236261
> +0100
> @@ -492,7 +492,7 @@
> } elsif ($path =~ m!^$privatedir/(\d+)$!) {
> my $vmid = $1;
> return ('rootdir', "$sid:rootdir/$vmid");
> - } elsif ($path =~ m!^$backupdir/([^/]+\.(tar|
> tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) {
> + } elsif ($path =~ m!^$backupdir/([^/]+\.(tar|
> tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo|tar\.zst|vma\.zst))$!) {
> my $name = $1;
> return ('iso', "$sid:backup/$name");
> }
> @@ -778,7 +778,7 @@
> $info = { volid => "$sid:vztmpl/$1", format => "t$2" };
>
> } elsif ($tt eq 'backup') {
> - next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.
> lzo|tgz|vma|vma\.gz|vma\.lzo))$!;
> + next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.
> lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!;
>
> $info = { volid => "$sid:backup/$1", format => $2 };
> }
> @@ -1295,8 +1295,14 @@
>
> die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
>
> - my $pid = open(my $fh, '-|', 'tar', 'tf', $archive) ||
> - die "unable to open file '$archive'\n";
> + my ($pid, $fh);
> + if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/ || $archive =~
> m/.tar.lzo$/) {
> + $pid = open($fh, '-|', 'tar', 'tf', $archive) ||
> + die "unable to open file '$archive'\n";
> + } elsif ($archive =~ m/.tar.zst$/) {
> + $pid = open($fh, '-|', "tar -I zstd -tf '$archive'") ||
> + die "unable to open file '$archive'\n";
> + }
>
> my $file;
> while (defined($file = <$fh>)) {
> @@ -1319,7 +1325,11 @@
> $raw .= "$output\n";
> };
>
> - PVE::Tools::run_command(['tar', '-xpOf', $archive, $file,
> '--occurrence'], outfunc => $out);
> + if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/ || $archive =~
> m/.tar.lzo$/) {
> + PVE::Tools::run_command(['tar', '-xpOf', $archive, $file,
> '--occurrence'], outfunc => $out);
> + } elsif ($archive =~ m/.tar.zst$/) {
> + PVE::Tools::run_command(['tar', '-I zstd', '-xpOf', $archive,
> $file, '--occurrence'], outfunc => $out);
> + }
>
> return wantarray ? ($raw, $file) : $raw;
> }
> @@ -1341,6 +1351,8 @@
> $uncomp = ["zcat", $archive];
> } elsif ($comp eq 'lzo') {
> $uncomp = ["lzop", "-d", "-c", $archive];
> + } elsif ($comp eq 'zst') {
> + $uncomp = ["zstd", "-d", $archive];
> } else {
> die "unknown compression method '$comp'\n";
> }
> @@ -1352,7 +1364,7 @@
> my $errstring;
> my $err = sub {
> my $output = shift;
> - if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~
> m/gzip: stdout: Broken pipe/) {
> + if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~
> m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: stdout: Broken pipe/) {
> $broken_pipe = 1;
> } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
> $errstring = "Failed to extract config from VMA archive:
> $output\n";
> @@ -1386,9 +1398,9 @@
>
> my $archive = abs_filesystem_path($cfg, $volid);
>
> - if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{
> 4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/) {
> + if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{
> 4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo|zst))?))$/)
> {
> return extract_vzdump_config_tar($archive,
> qr!^(\./etc/vzdump/(pct|vps)\.conf)$!);
> - } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{
> 2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/) {
> + } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{
> 2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/)
> {
> my $format;
> my $comp;
> if ($7 eq 'tgz') {
> --- /usr/share/perl5/PVE.org/VZDump.pm 2017-10-25 11:15:46.000000000
> +0200
> +++ /usr/share/perl5/PVE/VZDump.pm 2017-11-05 12:45:04.945033895 +0100
> @@ -56,7 +56,7 @@
> type => 'string',
> description => "Compress dump file.",
> optional => 1,
> - enum => ['0', '1', 'gzip', 'lzo'],
> + enum => ['0', '1', 'gzip', 'lzo', 'zstd'],
> default => '0',
> },
> pigz=> {
> @@ -66,6 +66,13 @@
> optional => 1,
> default => 0,
> },
> + zstd=> {
> + type => "integer",
> + description => "Use zstd instead of lzo/gzip/pigz when N>0.".
> + " N=1 uses half of cores, N>1 uses N as thread count.",
> + optional => 1,
> + default => 0,
> + },
> quiet => {
> type => 'boolean',
> description => "Be quiet.",
> @@ -716,6 +723,15 @@
> } else {
> return ('gzip', 'gz');
> }
> + } elsif ($opt_compress eq 'zstd') {
> + if ($opts->{zstd} > 0) {
> + my $zstd_threads = $opts->{zstd};
> + if ($zstd_threads == 1) {
> + my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
> + $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
> + }
> + return ("zstd -T${zstd_threads}", 'zst');
> + }
> } else {
> die "internal error - unknown compression option '$opt_compress'";
> }
> @@ -727,7 +743,7 @@
> my $bklist = [];
> foreach my $fn (<$dir/${bkname}-*>) {
> next if $exclude_fn && $fn eq $exclude_fn;
> - if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_
> (\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!) {
> + if ($fn =~ m!/(${bkname}-(\d{4})_(\d{2})_
> (\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?)))$!) {
> $fn = "$dir/$1"; # untaint
> my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2 - 1900);
> push @$bklist, [$fn, $t];
> @@ -985,7 +1001,7 @@
> debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
> unlink $d->[0];
> my $logfn = $d->[0];
> - $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
> + $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/\.log/;
> unlink $logfn;
> }
> }
> --- /usr/share/perl5/PVE/LXC.org/Create.pm 2017-10-17
> 15:11:19.000000000 +0200
> +++ /usr/share/perl5/PVE/LXC/Create.pm 2017-11-05 16:43:07.635271185
> +0100
> @@ -73,9 +73,16 @@
> $archive_fh->fcntl(Fcntl::F_SETFD(), $flags &
> ~(Fcntl::FD_CLOEXEC()));
> }
>
> - my $cmd = [@$userns_cmd, 'tar', 'xpf', $tar_input_file, '--totals',
> + my $cmd;
> + if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/ || $archive =~
> m/.tar.lzo$/) {
> + $cmd = [@$userns_cmd, 'tar', 'xpf', $tar_input_file, '--totals',
> @PVE::Storage::Plugin::COMMON_TAR_FLAGS,
> '-C', $rootdir];
> + } elsif ($archive =~ m/.tar.zst$/) {
> + $cmd = [@$userns_cmd, 'tar', '-I zstd', '-xpf', $tar_input_file,
> '--totals',
> + @PVE::Storage::Plugin::COMMON_TAR_FLAGS,
> + '-C', $rootdir];
> + }
>
> # skip-old-files doesn't have anything to do with time (old/new), but
> is
> # simply -k (annoyingly also called --keep-old-files) without the
> 'treat
> --- /usr/share/perl5/PVE/Storage.org/Plugin.pm 2017-10-17
> 15:02:17.000000000 +0200
> +++ /usr/share/perl5/PVE/Storage/Plugin.pm 2017-11-04
> 17:02:13.000000000 +0100
> @@ -388,7 +388,7 @@
> return ('vztmpl', $1);
> } elsif ($volname =~ m!^rootdir/(\d+)$!) {
> return ('rootdir', $1, $1);
> - } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.
> gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) {
> + } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.
> gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst)))$!) {
> my $fn = $1;
> if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) {
> return ('backup', $fn, $2);
>
> _______________________________________________
> pve-devel mailing list
> pve-devel at pve.proxmox.com
> https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
More information about the pve-devel
mailing list