[pve-devel] [PATCH pve-storage v4 1/3] recursively go through subdirs to find files
Fabian Grünbichler
f.gruenbichler at proxmox.com
Wed Jul 26 11:34:55 CEST 2023
On July 21, 2023 2:23 pm, Noel Ullreich wrote:
> This patch allows `get_subdir_files` to recursively call itself, so that
> subdirectories of set depth can be searched. We allow searching for
> isos, vztmpl and snippets but not backups.
>
> As a security measure, when parsing a given path, parent
> directories (`/../`) are forbidden.
>
> The feature is opt-in, i.e. the searchdepth is 0 by default. It can be
> changed via the API, the web interface and `pvesm` (see the other
> patches).
>
> Signed-off-by: Noel Ullreich <n.ullreich at proxmox.com>
> ---
> src/PVE/Storage.pm | 11 ++++++++
> src/PVE/Storage/Plugin.pm | 54 ++++++++++++++++++++++++---------------
> 2 files changed, 44 insertions(+), 21 deletions(-)
>
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index b99ed35..02abacf 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -113,6 +113,17 @@ our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst)/i;
>
> our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/;
>
> +our $INTERMEDIATE_SUBDIR_EXT_RE_3 = qr/(?:[0-9A-z\_\-\.]+\/)*[^\/]+/i;
> +
> +our $SUBDIR_ANY_FILEEXTENSION_EXT_RE_4 = qr/(?:[0-9A-z\_\-\.]+\/)*.+/i;
> +
1. common RE pitfall: use a-z instead of A-z, or a-zA-Z and drop the
`/i` - else this matches the characters [\]^` as well, since they sit
between 'Z' and 'a'. not all of them will work in practice, especially
the '\' one would require special handling - but we don't want them
anyhow ;)
2. these two should end with _0 , right? (they don't have any capturing
groups)
3. the first regular expression matches any string that doesn't contain
a '/', or any string containing slashes, where everything up to the last
slash must be from the restricted char set - the name is a bit weird
IMHO ;) maybe it would make more sense to keep the [^\/]+ part out of
it, and leave that at the call site? then it would actually just match
the intermediate sub-dirs, like the name implies (although I would still
drop the `_EXT`, it refers to a file extension, and that is the only
thing it *does not* match at the moment ;))..
4. the second RE basically matches any string of length >=1 ?
if we do the above change, a single RE should be enough (and the one
usage of $SUBDIR_ANY_FILEEXTENSION_EXT_RE_4 could also just be
$INTERMEDIATE_SUBDIR_RE_0[^\/]+ , like the other ones, but without any
specific extension/suffix..). it would also mean that parse_volname and
get_subdir_files accept the same things, instead of the latter allowing
more characters in subdir components (although in practice, its return
value will likely be filtered through parse_volume_id -> parse_volname
anyhow - not sure of *all* the code paths though..)
other than these, the patches look good to me and seem to work as
expected, the above is mostly (except for number 1) code
hygiene/style/naming :) symlinks are also handled transparently as
expected (e.g., no resolved path leaks into path/volume id/.., and the
size is correctly retrieved from the target, dangling symlinks are just
ignored).
> +# '..' is forbidden at the beginning, between two '/' and at the end
> +my $dots = quotemeta('..');
> +my $beginning = qr!^$dots/!;
> +my $between = qr!/$dots/!;
> +my $end = qr!/$dots$!;
> +our $forbidden_double_dots_re = qr!(?:$beginning|$between|$end)!;
> +
> # FIXME remove with PVE 8.0, add versioned breaks for pve-manager
> our $vztmpl_extension_re = $VZTMPL_EXT_RE_1;
>
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 9d3b1ae..8831fbb 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -621,6 +621,8 @@ sub parse_name_dir {
> sub parse_volname {
> my ($class, $volname) = @_;
>
> + die "volname must not contain parent directories '/../'\n" if $volname =~ $PVE::Storage::forbidden_double_dots_re;
> +
> if ($volname =~ m!^(\d+)/(\S+)/(\d+)/(\S+)$!) {
> my ($basedvmid, $basename) = ($1, $2);
> parse_name_dir($basename);
> @@ -631,9 +633,9 @@ sub parse_volname {
> my ($vmid, $name) = ($1, $2);
> my (undef, $format, $isBase) = parse_name_dir($name);
> return ('images', $name, $vmid, undef, undef, $isBase, $format);
> - } elsif ($volname =~ m!^iso/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!) {
> + } elsif ($volname =~ m!^iso/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::ISO_EXT_RE_0)$!) {
> return ('iso', $1);
> - } elsif ($volname =~ m!^vztmpl/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!) {
> + } elsif ($volname =~ m!^vztmpl/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::VZTMPL_EXT_RE_1)$!) {
> return ('vztmpl', $1);
> } elsif ($volname =~ m!^rootdir/(\d+)$!) {
> return ('rootdir', $1, $1);
> @@ -643,7 +645,7 @@ sub parse_volname {
> return ('backup', $fn, $2);
> }
> return ('backup', $fn);
> - } elsif ($volname =~ m!^snippets/([^/]+)$!) {
> + } elsif ($volname =~ m!^snippets/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3)$!) {
> return ('snippets', $1);
> }
>
> @@ -1212,28 +1214,33 @@ sub list_images {
> }
>
> # list templates ($tt = <iso|vztmpl|backup|snippets>)
> -my $get_subdir_files = sub {
> - my ($sid, $path, $tt, $vmid) = @_;
> +sub get_subdir_files {
> + my ($sid, $path, $tt, $scfg, $vmid, $remaining_depth) = @_;
> + my $storage_path = $scfg->{path};
> + my $content_dir = $scfg->{"content-dirs"}->{$tt} // $vtype_subdirs->{$tt};
>
> my $res = [];
>
> foreach my $fn (<$path/*>) {
> my $st = File::stat::stat($fn);
>
> - next if (!$st || S_ISDIR($st->mode));
> + next if (!$st);
> +
> + if (S_ISDIR($st->mode)) {
> + if ($remaining_depth) {
> + push @$res, get_subdir_files($sid, $fn, $tt, $scfg, $vmid, $remaining_depth-1);
> + }
> + next;
> + }
>
> my $info;
>
> if ($tt eq 'iso') {
> - next if $fn !~ m!/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!i;
> -
> + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::ISO_EXT_RE_0)/;
> $info = { volid => "$sid:iso/$1", format => 'iso' };
> -
> } elsif ($tt eq 'vztmpl') {
> - next if $fn !~ m!/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!;
> -
> + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::VZTMPL_EXT_RE_1)/;
> $info = { volid => "$sid:vztmpl/$1", format => "t$2" };
> -
> } elsif ($tt eq 'backup') {
> next if $fn !~ m!/([^/]+$PVE::Storage::BACKUP_EXT_RE_2)$!;
> my $original = $fn;
> @@ -1262,9 +1269,9 @@ my $get_subdir_files = sub {
>
> $info->{protected} = 1 if -e PVE::Storage::protection_file_path($original);
> } elsif ($tt eq 'snippets') {
> -
> + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::SUBDIR_ANY_FILEEXTENSION_EXT_RE_4)/;
> $info = {
> - volid => "$sid:snippets/". basename($fn),
> + volid => "$sid:snippets/$1", #basename($fn),
leftover comment? ;)
> format => 'snippet',
> };
> }
> @@ -1274,14 +1281,18 @@ my $get_subdir_files = sub {
>
> push @$res, $info;
> }
> -
> return $res;
> };
>
> +sub flatten {
> + map { ref eq 'ARRAY' ? flatten(@{$_}) : $_ } @_;
> +}
> +
> # If attributes are set on a volume, they should be included in the result.
> # See get_volume_attribute for a list of possible attributes.
> sub list_volumes {
> my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
> + my $max_depth = $scfg->{'scan-depth'} // 0;
>
> my $res = [];
> my $vmlist = PVE::Cluster::get_vmlist();
> @@ -1294,17 +1305,19 @@ sub list_volumes {
> my $path = $class->get_subdir($scfg, $type);
>
> if ($type eq 'iso' && !defined($vmid)) {
> - $data = $get_subdir_files->($storeid, $path, 'iso');
> + $data = get_subdir_files($storeid, $path, 'iso', $scfg, undef, $max_depth);
> } elsif ($type eq 'vztmpl'&& !defined($vmid)) {
> - $data = $get_subdir_files->($storeid, $path, 'vztmpl');
> + $data = get_subdir_files($storeid, $path , 'vztmpl', $scfg, undef, $max_depth);
> } elsif ($type eq 'backup') {
> - $data = $get_subdir_files->($storeid, $path, 'backup', $vmid);
> + $data = get_subdir_files($storeid, $path, 'backup', $scfg, $vmid, $max_depth);
> } elsif ($type eq 'snippets') {
> - $data = $get_subdir_files->($storeid, $path, 'snippets');
> + $data = get_subdir_files($storeid, $path, 'snippets', $scfg, undef, $max_depth);
> }
> }
>
> - next if !$data;
> + $data = [flatten($data)];
> +
> + next if !@$data[0];
>
> foreach my $item (@$data) {
> if ($type eq 'images' || $type eq 'rootdir') {
> @@ -1322,7 +1335,6 @@ sub list_volumes {
> } else {
> $item->{content} = $type;
> }
> -
> push @$res, $item;
> }
> }
> --
> 2.39.2
>
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel at lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
>
>
More information about the pve-devel
mailing list