[pve-devel] PATCH pve-zsync compress option for gz-compressed sync transfers

[Jochen Plumeyer]

Hi everybody,

according to my measurements, gz-compressing zfs syncs can reduce about 70%
of data traffic during pve-zsync operation.

Hence, the option "--compress" has been added in this patch.
If enabled, the existing traffic is directed through " | gzip -c " --SYNC
STREAM--> "gzip -d |" pipe commands.

Testing has been done, today we will deploy it in production.

Merging would be appreciated.

Cheers,

Jochen Plumeyer

diff --git a/pve-zsync b/pve-zsync
index 5c95955..57f722e 100755
--- a/pve-zsync
+++ b/pve-zsync
@@ -64,6 +64,7 @@ if (defined($command) && $command ne 'help' && $command ne
'printpod') {
     check_bin ('zfs');
     check_bin ('ssh');
     check_bin ('scp');
+    check_bin ('gzip');
 }
 
 $SIG{TERM} = $SIG{QUIT} = $SIG{PIPE} = $SIG{HUP} = $SIG{KILL} = $SIG{INT} =
sub {
@@ -229,6 +230,7 @@ sub parse_argv {
 	dest => undef,
 	source => undef,
 	verbose => undef,
+	compress => undef,
 	limit => undef,
 	maxsnap => undef,
 	name => undef,
@@ -245,6 +247,7 @@ sub parse_argv {
 	'dest=s' => \$param->{dest},
 	'source=s' => \$param->{source},
 	'verbose' => \$param->{verbose},
+	'compress' => \$param->{compress},
 	'limit=i' => \$param->{limit},
 	'maxsnap=i' => \$param->{maxsnap},
 	'name=s' => \$param->{name},
@@ -446,6 +449,7 @@ sub format_job {
     $text .= " --limit $job->{limit}" if $job->{limit};
     $text .= " --method $job->{method}";
     $text .= " --verbose" if $job->{verbose};
+    $text .= " --compress" if $job->{compress};
     $text .= " --source-user $job->{source_user}";
     $text .= " --dest-user $job->{dest_user}";
     $text .= " --properties" if $job->{properties};
@@ -967,6 +971,8 @@ sub send_image {
     }
     push @$cmd, '--', "$source->{all}\@$source->{new_snap}";
 
+    push @$cmd, \'|', 'gzip', '-c' if $param->{compress};
+
     if ($param->{limit}){
 	my $bwl = $param->{limit}*1024;
 	push @$cmd, \'|', 'cstream', '-t', $bwl;
@@ -977,8 +983,10 @@ sub send_image {
 
     push @$cmd, \'|';
     push @$cmd, 'ssh', '-o', 'BatchMode=yes',
"$param->{dest_user}\@$dest->{ip}", '--' if $dest->{ip};
+    push @$cmd, \'"', 'gzip', '-d', \'|' if $param->{compress};
     push @$cmd, 'zfs', 'recv', '-F', '--';
     push @$cmd, "$target";
+    push @$cmd, \'"' if $param->{compress};
 
     eval {
 	run_cmd($cmd)
@@ -1172,6 +1180,10 @@ $PROGNAME sync -dest <string> -source <string>
[OPTIONS]\n
 
 		print out the sync progress.
 
+	-compress   boolean
+
+		use gzip compression for sync transfer
+
 	-properties	boolean
 
 		Include the dataset's properties in the stream.