[pve-devel] [PATCH v2] Added parallel gzip compress support for vzdump with pigz.

[Thomas Lamprecht]

Added two options in vzdump.conf to controll pigz. 'usepigz' (default to false) to enable it
and 'pigzthreads' (defaults to #cores/2) to control how many threads pigz use.
To use it select gzip in the web interface and set usepigz: 1 in /etc/vzdump.conf

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---

Changes since v1:
* fixed indentation in code
* split commit message

 PVE/VZDump.pm     | 32 +++++++++++++++++++++++++++++---
 debian/control.in |  2 +-
 vzdump.conf       |  2 ++
 3 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 61341ca..bdc1f4e 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -189,6 +189,7 @@ sub read_vzdump_defaults {
 	stopwait => 10, # 10 minutes
 	mode => 'snapshot',
 	maxfiles => 1, 
+	usepigz => 0,
     };
 
     my $fh = IO::File->new ("<$fn");
@@ -225,6 +226,11 @@ sub read_vzdump_defaults {
 	    $res->{'exclude-path'} = PVE::Tools::split_args($1); 
 	} elsif ($line =~ m/mode:\s*(stop|snapshot|suspend)\s*$/) {
 	    $res->{mode} = $1;
+	} elsif ($line =~ m/usepigz:\s*(0|1|false|true)\s*$/) {
+	    $res->{usepigz} = $1 =~ /1|true/;
+	} elsif($line =~ m/pigzthreads:\s*(\d+)\s*/)
+	{
+	    $res->{pigzthreads} = $1;
 	} else {
 	    debugmsg ('warn', "unable to parse configuration file '$fn' - error at line " . $., undef, 1);
 	}
@@ -678,14 +684,22 @@ sub run_hook_script {
 }
 
 sub compressor_info {
-    my ($opt_compress) = @_;
+    my ($opts) = @_;
+    my $opt_compress = $opts->{compress};
 
     if (!$opt_compress || $opt_compress eq '0') {
 	return undef;
     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 	return ('lzop', 'lzo');
     } elsif ($opt_compress eq 'gzip') {
-	return ('gzip', 'gz');
+	if ($opts->{usepigz}) {
+	    # As default use int((#cores + 1)/2), we need #cores+1 for the case that #cores = 1
+	    my $pigz_thread_num = ($opts->{pigzthreads}) ? $opts->{pigzthreads} : int((int(`/usr/bin/nproc`) + 1) / 2);
+	    $pigz_thread_num = $1 if $pigz_thread_num =~ /(\d*)/; # untaint
+	    return ('pigz -p '.$pigz_thread_num, 'gz');
+	} else {
+	    return ('gzip', 'gz');
+	}
     } else {
 	die "internal error - unknown compression option '$opt_compress'";
     }
@@ -748,7 +762,7 @@ sub exec_backup_task {
 	my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 
 	my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
-	my ($comp, $comp_ext) = compressor_info($opts->{compress});
+	my ($comp, $comp_ext) = compressor_info($opts);
 	if ($comp && $comp_ext) {
 	    $ext .= ".${comp_ext}";
 	}
@@ -1123,6 +1137,18 @@ my $confdesc = {
 	enum => ['0', '1', 'gzip', 'lzo'],
 	default => 'lzo',
     },
+    usepigz=> {
+	type => "boolean",
+	description => "Uses pigz instead of gzip when set to true",
+	optional => 1,
+	default => 'false',
+    },
+    pigzthreads => {
+	type => "integer",
+	description => "Number of threads pigz should use. Default uses half the detected processors.",
+	optional => 1,
+	minimum => 1,
+    },
     quiet => {
 	type => 'boolean',
 	description => "Be quiet.",
diff --git a/debian/control.in b/debian/control.in
index 7b78973..d8094d4 100644
--- a/debian/control.in
+++ b/debian/control.in
@@ -3,7 +3,7 @@ Version: @VERSION at -@PACKAGERELEASE@
 Section: admin
 Priority: optional
 Architecture: amd64
-Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, spiceterm, libuuid-perl, hdparm, gdisk, librados2-perl, pve-firewall, novnc-pve, libev-perl, systemd, pve-ha-manager, pve-container
+Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, spiceterm, libuuid-perl, hdparm, gdisk, librados2-perl, pve-firewall, novnc-pve, libev-perl, systemd, pve-ha-manager, pve-container, pigz
 Conflicts: netcat-openbsd, vzdump
 Replaces: vzdump
 Provides: vzdump
diff --git a/vzdump.conf b/vzdump.conf
index b5ff34f..1c21036 100644
--- a/vzdump.conf
+++ b/vzdump.conf
@@ -12,3 +12,5 @@
 #maxfiles: N
 #script: FILENAME
 #exclude-path: PATHLIST
+#usepigz: BOOLEAN
+#pigzthreads: INTEGER
-- 
2.1.4