[pve-devel] [PATCH v3] Added parallel compress support for vzdump with pigz.

[Thomas Lamprecht]

Added a vzdump.conf option to controll gzip compression.
When 'pigz' (defaults to 0) is >0, pigz support is enabled.
When the pigz option equals 1 pigz uses #cores/2 threads,
else it spawns N threads. To use it select gzip in the web
interface and set the aproppriate option in /etc/vzdump.conf

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
Changes since v2:
* only use one option to control pigz behaviour
* use POSIX sysconf call to get number of processors

Changes since v1:
* fixed indentation in code
* split commit message

PVE/VZDump.pm     | 24 +++++++++++++++++++++---
 debian/control.in |  2 +-
 vzdump.conf       |  1 +
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 61341ca..31b6966 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -189,6 +189,7 @@ sub read_vzdump_defaults {
 	stopwait => 10, # 10 minutes
 	mode => 'snapshot',
 	maxfiles => 1, 
+	pigz => 0,
     };
 
     my $fh = IO::File->new ("<$fn");
@@ -225,6 +226,8 @@ sub read_vzdump_defaults {
 	    $res->{'exclude-path'} = PVE::Tools::split_args($1); 
 	} elsif ($line =~ m/mode:\s*(stop|snapshot|suspend)\s*$/) {
 	    $res->{mode} = $1;
+	} elsif($line =~ m/pigz:\s*(\d+)\s*/) {
+	    $res->{pigz} = $1;
 	} else {
 	    debugmsg ('warn', "unable to parse configuration file '$fn' - error at line " . $., undef, 1);
 	}
@@ -678,14 +681,22 @@ sub run_hook_script {
 }
 
 sub compressor_info {
-    my ($opt_compress) = @_;
+    my ($opts) = @_;
+    my $opt_compress = $opts->{compress};
 
     if (!$opt_compress || $opt_compress eq '0') {
 	return undef;
     } elsif ($opt_compress eq '1' || $opt_compress eq 'lzo') {
 	return ('lzop', 'lzo');
     } elsif ($opt_compress eq 'gzip') {
-	return ('gzip', 'gz');
+	if ($opts->{pigz}>0) {
+	    # As default use int((#cores + 1)/2), we need #cores+1 for the case that #cores = 1
+	    my $cores = POSIX::sysconf(84);
+	    my $pigz_threads = ($opts->{pigz}>1) ? $opts->{pigz}: int(($cores+1)/2);
+	    return ('pigz -p '.$pigz_threads, 'gz');
+	} else {
+	    return ('gzip', 'gz');
+	}
     } else {
 	die "internal error - unknown compression option '$opt_compress'";
     }
@@ -748,7 +759,7 @@ sub exec_backup_task {
 	my $logfile = $task->{logfile} = "$opts->{dumpdir}/$basename.log";
 
 	my $ext = $vmtype eq 'qemu' ? '.vma' : '.tar';
-	my ($comp, $comp_ext) = compressor_info($opts->{compress});
+	my ($comp, $comp_ext) = compressor_info($opts);
 	if ($comp && $comp_ext) {
 	    $ext .= ".${comp_ext}";
 	}
@@ -1123,6 +1134,13 @@ my $confdesc = {
 	enum => ['0', '1', 'gzip', 'lzo'],
 	default => 'lzo',
     },
+    pigz=> {
+	type => "integer",
+	description => "Uses pigz instead of gzip when N>0.".
+	    " N=1 uses half of cores, N>1 uses N as thread count.",
+	optional => 1,
+	default => 0,
+    },
     quiet => {
 	type => 'boolean',
 	description => "Be quiet.",
diff --git a/debian/control.in b/debian/control.in
index 7b78973..d8094d4 100644
--- a/debian/control.in
+++ b/debian/control.in
@@ -3,7 +3,7 @@ Version: @VERSION at -@PACKAGERELEASE@
 Section: admin
 Priority: optional
 Architecture: amd64
-Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, spiceterm, libuuid-perl, hdparm, gdisk, librados2-perl, pve-firewall, novnc-pve, libev-perl, systemd, pve-ha-manager, pve-container
+Depends: perl (>= 5.10.0-19), libtimedate-perl, libauthen-pam-perl, libintl-perl, rsync, libjson-perl, liblockfile-simple-perl, vncterm, qemu-server (>= 1.1-1), libwww-perl (>= 6.04-1), libnet-http-perl (>= 6.06-1), libhttp-daemon-perl, wget, libnet-dns-perl, vlan, ifenslave-2.6 (>= 1.1.0-10), liblinux-inotify2-perl, debconf (>= 0.5) | debconf-2.0, netcat-traditional, pve-cluster (>= 1.0-29), libpve-common-perl, libpve-storage-perl, libterm-readline-gnu-perl, libpve-access-control (>= 3.0-2), libio-socket-ssl-perl, libfilesys-df-perl, libfile-readbackwards-perl, libfile-sync-perl, cstream, postfix | mail-transport-agent, libxml-parser-perl, lzop, dtach, libanyevent-perl, liburi-perl, logrotate, libanyevent-http-perl, apt-transport-https, libapt-pkg-perl, libcrypt-ssleay-perl, liblwp-protocol-https-perl, spiceterm, libuuid-perl, hdparm, gdisk, librados2-perl, pve-firewall, novnc-pve, libev-perl, systemd, pve-ha-manager, pve-container, pigz
 Conflicts: netcat-openbsd, vzdump
 Replaces: vzdump
 Provides: vzdump
diff --git a/vzdump.conf b/vzdump.conf
index b5ff34f..ee0ac4b 100644
--- a/vzdump.conf
+++ b/vzdump.conf
@@ -12,3 +12,4 @@
 #maxfiles: N
 #script: FILENAME
 #exclude-path: PATHLIST
+#pigz: N:
\ No newline at end of file
-- 
2.1.4