[pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs

[Fabian Grünbichler]

On July 31, 2023 10:39 am, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl at proxmox.com>
> ---
>  src/PVE/API2/Storage/Status.pm | 17 +++++++++++++++--
>  src/PVE/Storage.pm             |  6 ++++++
>  2 files changed, 21 insertions(+), 2 deletions(-)
> 
> diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
> index 2aaeff6..1d73c96 100644
> --- a/src/PVE/API2/Storage/Status.pm
> +++ b/src/PVE/API2/Storage/Status.pm
> @@ -23,6 +23,8 @@ use PVE::Storage;
>  
>  use base qw(PVE::RESTHandler);
> 
> +our $KNOWN_COMPRESSION_FORMATS = ['zst', 'gz', 'lzo'];

might want a comment here that adding new formats means handling them in
decompressor_info below, and vice-versa?

also, this is just another way to write PVE::Storage::Plugin::COMPRESSOR_RE

maybe they could be unified to avoid accidentally desyncing them?
technically, the RE could be used as 'pattern' in the API schema, but
the generated docs are less human-friendly for patterns compared to
enums..

> +
>  __PACKAGE__->register_method ({
>      subclass => "PVE::API2::Storage::PruneBackups",
>      path => '{storage}/prunebackups',
> @@ -578,6 +580,12 @@ __PACKAGE__->register_method({
>  		requires => 'checksum-algorithm',
>  		optional => 1,
>  	    },
> +	    compression => {
> +		description => "Decompress the downloaded file using specified compression algorithm",
> +		type => 'string',
> +		enum => $KNOWN_COMPRESSION_FORMATS,
> +		optional => 1,
> +	    },
>  	    'checksum-algorithm' => {
>  		description => "The algorithm to calculate the checksum of the file.",
>  		type => 'string',
> @@ -604,7 +612,7 @@ __PACKAGE__->register_method({
>  
>  	my $cfg = PVE::Storage::config();
>  
> -	my ($node, $storage) = $param->@{'node', 'storage'};
> +	my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
>  	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
>  
>  	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
> @@ -642,13 +650,18 @@ __PACKAGE__->register_method({
>  	    http_proxy => $dccfg->{http_proxy},
>  	};
>  
> -	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
> +	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm' };
>  	if ($checksum) {
>  	    $opts->{"${checksum_algorithm}sum"} = $checksum;
>  	    $opts->{hash_required} = 1;
>  	}
>  
>  	my $worker = sub {
> +	    if ($compression) {
> +		die "decompression not supported for $content\n" if $content ne 'iso';
> +		my $info = PVE::Storage::decompressor_info('iso', $compression);
> +		$opts->{decompression_command} = $info->{decompressor};

note: $info->{decompressor} might be undef, should probably be caught
here, else decompression will be silently skipped

> +	    }
>  	    PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
>  	};
>  
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index a4d85e1..cb70113 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -1531,6 +1531,12 @@ sub decompressor_info {
>  	    lzo => ['lzop', '-d', '-c'],
>  	    zst => ['zstd', '-q', '-d', '-c'],
>  	},
> +	iso => {
> +	    # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
> +	    gz => ['zcat'],
> +	    lzo => ['lzop', '-d', '-c'],
> +	    zst => ['zstd', '-q', '-d', '-c'],
> +	},
>      };
>  
>      die "ERROR: archive format not defined\n"
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel at lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
>