[pve-devel] [PATCH v3 storage 1/2] fix #1710: add retrieve method for storage

[Lorenz Stechauner]

Users are now able to download/retrieve any .iso/... file onto their
storages and verify file integrity with checksums.

Signed-off-by: Lorenz Stechauner <l.stechauner at proxmox.com>
---
 PVE/API2/Storage/Status.pm | 158 +++++++++++++++++++++++++++++++++++--
 PVE/Storage.pm             |  10 +++
 2 files changed, 163 insertions(+), 5 deletions(-)

diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm
index 897b4a7..8388714 100644
--- a/PVE/API2/Storage/Status.pm
+++ b/PVE/API2/Storage/Status.pm
@@ -5,6 +5,8 @@ use warnings;
 
 use File::Path;
 use File::Basename;
+use HTTP::Request;
+use LWP::UserAgent;
 use PVE::Tools;
 use PVE::INotify;
 use PVE::Cluster;
@@ -412,11 +414,7 @@ __PACKAGE__->register_method ({
 	my $size = -s $tmpfilename;
 	die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
 
-	my $filename = $param->{filename};
-
-	chomp $filename;
-	$filename =~ s/^.*[\/\\]//;
-	$filename =~ s/[^-a-zA-Z0-9_.]/_/g;
+	my $filename = PVE::Storage::normalize_content_filename($param->{filename});
 
 	my $path;
 
@@ -497,4 +495,154 @@ __PACKAGE__->register_method ({
 	return $upid;
    }});
 
+__PACKAGE__->register_method({
+    name => 'retrieve',
+    path => '{storage}/retrieve',
+    method => 'POST',
+    description => "Download templates and ISO images by using an URL.",
+    proxyto => 'node',
+    permissions => {
+	check => [ 'and',
+	    ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
+	    ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
+	],
+    },
+    protected => 1,
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    storage => get_standard_option('pve-storage-id'),
+	    url => {
+		description => "The URL to retrieve the file from.",
+		type => 'string',
+	    },
+	    content => {
+		description => "Content type.",
+		type => 'string', format => 'pve-storage-content',
+	    },
+	    filename => {
+		description => "The name of the file to create. Alternatively the file name given by the server will be used.",
+		type => 'string',
+	    },
+	    checksum => {
+		description => "The expected checksum of the file.",
+		type => 'string',
+		requires => 'checksumalg',
+		optional => 1,
+	    },
+	    checksumalg => {
+		description => "The algorithm to claculate the checksum of the file.",
+		type => 'string', enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
+		requires => 'checksum',
+		optional => 1,
+	    },
+	    insecure => {
+		description => "Allow TLS certificates to be invalid.",
+		type => 'boolean',
+		optional => 1,
+	    }
+	},
+    },
+    returns => { type => "string" },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $user = $rpcenv->get_user();
+
+	my $cfg = PVE::Storage::config();
+
+	my $node = $param->{node};
+	my $storage = $param->{storage};
+	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
+
+	die "can't upload to storage type '$scfg->{type}'"
+	    if !defined($scfg->{path});
+
+	my $content = $param->{content};
+
+	my $url = $param->{url};
+
+	die "invalid https or http url"
+	    if $url !~ qr!^https?://!;
+
+	my $checksum = $param->{checksum};
+	my $hash_alg = $param->{checksumalg};
+
+	my $filename = PVE::Storage::normalize_content_filename($param->{filename});
+
+	my $path;
+
+	# MIME type is checked in front end only
+	# this check is omitted here intentionally and replaced by file extension check
+	if ($content eq 'iso') {
+	    if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
+		raise_param_exc({ filename => "missing '.iso' or '.img' extension" });
+	    }
+	    $path = PVE::Storage::get_iso_dir($cfg, $storage);
+	} elsif ($content eq 'vztmpl') {
+	    if ($filename !~ m![^/]+\.tar\.[gx]z$!) {
+		raise_param_exc({ filename => "missing '.tar.gz' or '.tar.xz' extension" });
+	    }
+	    $path = PVE::Storage::get_vztmpl_dir($cfg, $storage);
+	} else {
+	    raise_param_exc({ content => "upload content type '$content' not allowed" });
+	}
+
+	die "storage '$storage' does not support '$content' content"
+	    if !$scfg->{content}->{$content};
+
+	my $dest = "$path/$filename";
+
+	PVE::Storage::activate_storage($cfg, $storage);
+	File::Path::make_path($path);
+
+	# -L follows redirects
+	# -f silent fail on error
+	my @curlcmd = ('curl', '-L', '-o', $dest, '-f');
+	push @curlcmd, '--insecure'
+	    if $param->{insecure};
+
+	my $cmd = [@curlcmd, $url];
+
+	my $cmd_check = [ [ 'echo', $checksum, $dest ], [ "${hash_alg}sum", '-c' ] ];
+	my $cmd_check_flat = [ 'echo', $checksum, $dest, '|', "${hash_alg}sum", '-c'  ];  # only used for logging
+
+	my $worker = sub {
+	    my $upid = shift;
+
+	    print "starting file download from: $url\n";
+	    print "target node: $node\n";
+	    print "target storage: $storage\n";
+	    print "target file: $dest\n";
+	    print "command: " . join(' ', @$cmd) . "\n";
+
+	    eval { PVE::Tools::run_command($cmd, errmsg => 'download failed'); };
+	    if (my $err = $@) {
+		unlink $dest;
+		die $err;
+	    }
+	    print "finished file download successfully\n";
+
+	    if ($checksum) {
+		print "validating checksum...\n";
+		print "expected $hash_alg checksum is: $checksum\n";
+		print cmd_check print "checksum validation command: " . join(' ', @$cmd_check_flat) . "\n";
+
+		eval { PVE::Tools::run_command($cmd_check, errmsg => 'checksum mismatch'); };
+		if (my $err = $@) {
+		    unlink $dest;
+		    die $err;
+		}
+		print "validated checksum successfully\n";
+	    }
+	};
+
+	my $upid = $rpcenv->fork_worker('imgdownload', $storage, $user, $worker);
+
+	return $upid;
+    }});
+
 1;
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 122c3e9..d57fd43 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1931,4 +1931,14 @@ sub assert_sid_unused {
     return undef;
 }
 
+sub normalize_content_filename {
+    my ($filename) = @_;
+
+    chomp $filename;
+    $filename =~ s/^.*[\/\\]//;
+    $filename =~ s/[^-a-zA-Z0-9_.]/_/g;
+
+    return $filename;
+}
+
 1;
-- 
2.20.1