[pve-devel] [PATCH] Implements ZFSLocalPlugin.pm to use Zfs with volumes locally. You can snapshot on the volumes but only in a linear way.

Adrian Costin adrian at goat.fish
Tue Jan 20 18:29:59 CET 2015


I’ve been using a similar plugin for local zfs, basically a stripped down version of the ZFSPlugin.pm, however I’ve been thinking if there’s a way to implement this as a LunCmd for the actual ZFS Plugin instead of a separate plugin.

Do you think it’s worth pursuing or is a separate plugin better?

Best regards,
Adrian Costin


> On Jan 20, 2015, at 4:37 PM, Wolfgang Link <wolfgang at linksystems.org> wrote:
> 
> 
> Signed-off-by: Wolfgang Link <wolfgang at linksystems.org>
> ---
> PVE/Storage.pm                |    2 +
> PVE/Storage/Makefile          |    2 +-
> PVE/Storage/ZFSLocalPlugin.pm |  517 +++++++++++++++++++++++++++++++++++++++++
> 3 files changed, 520 insertions(+), 1 deletion(-)
> create mode 100644 PVE/Storage/ZFSLocalPlugin.pm
> 
> diff --git a/PVE/Storage.pm b/PVE/Storage.pm
> index 3cc9581..b978b66 100755
> --- a/PVE/Storage.pm
> +++ b/PVE/Storage.pm
> @@ -28,6 +28,7 @@ use PVE::Storage::SheepdogPlugin;
> use PVE::Storage::ISCSIDirectPlugin;
> use PVE::Storage::GlusterfsPlugin;
> use PVE::Storage::ZFSPlugin;
> +use PVE::Storage::ZFSLocalPlugin;
> 
> # load and initialize all plugins
> PVE::Storage::DirPlugin->register();
> @@ -39,6 +40,7 @@ PVE::Storage::SheepdogPlugin->register();
> PVE::Storage::ISCSIDirectPlugin->register();
> PVE::Storage::GlusterfsPlugin->register();
> PVE::Storage::ZFSPlugin->register();
> +PVE::Storage::ZFSLocalPlugin->register();
> PVE::Storage::Plugin->init();
> 
> my $UDEVADM = '/sbin/udevadm';
> diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
> index 919c486..475f757 100644
> --- a/PVE/Storage/Makefile
> +++ b/PVE/Storage/Makefile
> @@ -1,4 +1,4 @@
> -SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPlugin.pm
> +SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPlugin.pm ZFSLocalPlugin.pm
> 
> .PHONY: install
> install:
> diff --git a/PVE/Storage/ZFSLocalPlugin.pm b/PVE/Storage/ZFSLocalPlugin.pm
> new file mode 100644
> index 0000000..df211f7
> --- /dev/null
> +++ b/PVE/Storage/ZFSLocalPlugin.pm
> @@ -0,0 +1,517 @@
> +package PVE::Storage::ZFSLocalPlugin;
> +
> +use strict;
> +use warnings;
> +use IO::File;
> +use POSIX;
> +use PVE::Tools qw(run_command);
> +use PVE::Storage::Plugin;
> +
> +use base qw(PVE::Storage::Plugin);
> +
> +sub zfs_request {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +
> +    my $cmdmap;
> +    my $zfscmd;
> +    my $msg;
> +
> +    $timeout = 5 if !$timeout;
> +
> +    if ($method eq 'zpool_list') {
> +	$zfscmd = 'zpool';
> +	$method = 'list',
> +    } else {
> +	$zfscmd = 'zfs';
> +    }
> +
> +    my $cmd = [$zfscmd, $method, @params];
> +
> +    $msg = '';
> +
> +    my $output = sub {
> +        my $line = shift;
> +        $msg .= "$line\n";
> +    };
> +
> +    run_command($cmd, outfunc => $output, timeout => $timeout);
> +
> +    return $msg;
> +}
> +
> +sub zfs_parse_size {
> +    my ($text) = @_;
> +
> +    return 0 if !$text;
> +
> +    if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
> +    my ($size, $reminder, $unit) = ($1, $2, $3);
> +    return $size if !$unit;
> +    if ($unit eq 'K') {
> +        $size *= 1024;
> +    } elsif ($unit eq 'M') {
> +        $size *= 1024*1024;
> +    } elsif ($unit eq 'G') {
> +        $size *= 1024*1024*1024;
> +    } elsif ($unit eq 'T') {
> +        $size *= 1024*1024*1024*1024;
> +    }
> +
> +    if ($reminder) {
> +        $size = ceil($size);
> +    }
> +    return $size;
> +    } else {
> +    return 0;
> +    }
> +}
> +
> +sub zfs_get_pool_stats {
> +    my ($scfg) = @_;
> +
> +    my $available = 0;
> +    my $used = 0;
> +
> +    my $text = '';
> +    eval {
> +	$text = zfs_request($scfg, undef, 'get', '-o', 'value', '-Hp','available,used', $scfg->{pool});
> +    };
> +    warn $@ if $@;
> +
> +    my @lines = split /\n/, $text;
> +
> +    if($lines[0] =~ /^(\d+)$/) {
> +	$available = $1;
> +    }
> +
> +    if($lines[1] =~ /^(\d+)$/) {
> +	$used = $1;
> +    }
> +
> +    return ($available, $used);
> +}
> +
> +sub zfs_parse_zvol_list {
> +    my ($text) = @_;
> +
> +    my $list = ();
> +
> +    return $list if !$text;
> +
> +    my @lines = split /\n/, $text;
> +    foreach my $line (@lines) {
> +	if ($line =~ /^(.+)\s+([a-zA-Z0-9\.]+|\-)\s+(.+)$/) {
> +	    my $zvol = {};
> +	    my @parts = split /\//, $1;
> +	    my $name = pop @parts;
> +	    my $pool = join('/', @parts);
> +
> +	    $zvol->{pool} = $pool;
> +	    $zvol->{name} = $name;
> +	    $zvol->{size} = zfs_parse_size($2);
> +	    if ($3 !~ /^-$/) {
> +		$zvol->{origin} = $3;
> +	    }
> +	    push @$list, $zvol;
> +	}
> +    }
> +
> +    return $list;
> +}
> +
> +sub zfs_get_zvol_size {
> +    my ($scfg, $zvol) = @_;
> +
> +    my $text = '';
> +    eval {
> +	$text = zfs_request($scfg, undef, 'get', '-Hp', 'volsize', "$scfg->{pool}/$zvol");
> +    };
> +    warn $@ if $@;
> +    if($text =~ /volsize\s(\d+)/){
> +	return $1;
> +    }
> +
> +    die "Could not get zvol size";
> +}
> +
> +sub zfs_create_zvol {
> +    my ($scfg, $zvol, $size) = @_;
> +
> +    my $blocksize = "8kb";
> +    if ($scfg->{blocksize}) {
> +	$blocksize =  $scfg->{blocksize};
> +    }
> +
> +    eval {
> +	if ($scfg->{sparse}) {
> +	    zfs_request($scfg, undef, 'create', '-s', '-b', $blocksize, '-V', "${size}k", "$scfg->{pool}/$zvol");
> +	} else {
> +	    zfs_request($scfg, undef, 'create', '-b', $blocksize, '-V', "${size}k", "$scfg->{pool}/$zvol");
> +    }
> +    };
> +    warn $@ if $@;
> +}
> +
> +sub zfs_delete_zvol {
> +    my ($scfg, $zvol) = @_;
> +    eval {
> +	zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol");
> +    };
> +    warn $@ if $@;
> +}
> +
> +sub zfs_list_zvol {
> +    my ($scfg) = @_;
> +
> +    my $text = '';
> +    eval {
> +	$text = zfs_request($scfg, 5, 'list', '-o', 'name,volsize,origin', '-t', 'volume', '-Hr');
> +    };
> +    warn $@ if $@;
> +
> +    my $zvols = zfs_parse_zvol_list($text);
> +    return undef if !$zvols;
> +
> +    my $list = ();
> +    foreach my $zvol (@$zvols) {
> +	my @values = split('/', $zvol->{name});
> +	my $image = pop @values;
> +	my $pool =  $zvol->{pool};
> +
> +	next if $image !~ m/^((vm|base)-(\d+)-\S+)$/;
> +	my $owner = $3;
> +
> +	my $parent = $zvol->{origin};
> +	if($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/){
> +	    $parent = $1;
> +	}
> +
> +	$list->{$pool}->{$image} = {
> +	    name => $image,
> +	    size => $zvol->{size},
> +	    parent => $parent,
> +	    format => 'raw',
> +            vmid => $owner
> +        };
> +    }
> +
> +    return $list;
> +}
> +
> +# Configuration
> +
> +sub type {
> +    return 'zfslocal';
> +}
> +
> +sub plugindata {
> +    return {
> +    content => [ {images => 1}, { images => 1 }],
> +    };
> +}
> +
> +sub properties {
> +    return {
> +    };
> +}
> +
> +sub options {
> +    return {
> +    nodes => { optional => 1 },
> +    disable => { optional => 1 },
> +    pool => { fixed => 1 },
> +    blocksize => { optional => 1 },
> +    nowritecache => { optional => 1 },
> +    sparse => { optional => 1 },
> +    content => { optional => 1 },
> +    };
> +}
> +
> +# Storage implementation
> +
> +sub parse_volname {
> +    my ($class, $volname) = @_;
> +
> +    if ($volname =~ m/^(((base|vm)-(\d+)-\S+)\_)?((base)?(vm)?-(\d+)-\S+)$/) {
> +	return ('images', $5, $8, $2, $4, $6);
> +    }
> +
> +    die "unable to parse zfs volume name '$volname'\n";
> +}
> +
> +sub path {
> +    my ($class, $scfg, $volname) = @_;
> +
> +    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
> +
> +    my $path = "/dev/zvol/$scfg->{pool}/$volname";
> +
> +    return ($path, $vmid, $vtype);
> +}
> +
> +my $find_free_diskname = sub {
> +    my ($storeid, $scfg, $vmid) = @_;
> +
> +    my $name = undef;
> +    my $volumes = zfs_list_zvol($scfg);
> +
> +    my $disk_ids = {};
> +    my $dat = $volumes->{$scfg->{pool}};
> +
> +    foreach my $image (keys %$dat) {
> +        my $volname = $dat->{$image}->{name};
> +        if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
> +            $disk_ids->{$2} = 1;
> +        }
> +    }
> +
> +    for (my $i = 1; $i < 100; $i++) {
> +        if (!$disk_ids->{$i}) {
> +            return "vm-$vmid-disk-$i";
> +        }
> +    }
> +
> +    die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
> +};
> +sub create_base {
> +    my ($class, $storeid, $scfg, $volname) = @_;
> +
> +    my $snap = '__base__';
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
> +        $class->parse_volname($volname);
> +
> +    die "create_base not possible with base image\n" if $isBase;
> +
> +    my $newname = $name;
> +    $newname =~ s/^vm-/base-/;
> +
> +    my $newvolname = $basename ? "$basename/$newname" : "$newname";
> +    eval {
> +	zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
> +        };
> +    warn $@ if $@;
> +    my $running  = undef; #fixme : is create_base always offline ?
> +
> +    $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
> +
> +    return $newvolname;
> +}
> +
> +sub clone_image {
> +    my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
> +
> +    $snap ||= '__base__';
> +
> +    my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
> +        $class->parse_volname($volname);
> +
> +    die "clone_image only works on base images\n" if !$isBase;
> +
> +    my $name = &$find_free_diskname($storeid, $scfg, $vmid);
> +
> +    warn "clone $volname: $basename to $name\n";
> +    eval {
> +	zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$basename\_$name");
> +    };
> +    warn $@ if $@;
> +    return "$basename\_$name";
> +}
> +
> +sub alloc_image {
> +    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
> +
> +    die "unsupported format '$fmt'" if $fmt ne 'raw';
> +
> +    die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
> +    if $name && $name !~ m/^vm-$vmid-/;
> +
> +    $name = &$find_free_diskname($storeid, $scfg, $vmid) if !$name;
> +    eval {
> +	zfs_create_zvol($scfg, $name, $size);
> +        };
> +    warn $@ if $@;
> +
> +    run_command ("udevadm trigger --subsystem-match block");
> +    run_command ("udevadm settle --timeout 3");
> +
> +    my $file = "/dev/zvol/".$scfg->{pool}."/$name" ;
> +    for (my $i = 5; $i > 0; $i--) {
> +	last if -e $file;
> + 	Time::HiRes::usleep(100);
> +    }
> +
> +    return $name;
> +}
> +
> +sub free_image {
> +    my ($class, $storeid, $scfg, $volname, $isBase) = @_;
> +
> +    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
> +
> +    eval {
> +        zfs_delete_zvol($scfg, $volname);
> +    };
> +    warn $@ if $@;
> +
> +    return undef;
> +}
> +
> +sub list_images {
> +    my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
> +
> +    $cache->{zfs} = zfs_list_zvol($scfg) if !$cache->{zfs};
> +    my $zfspool = $scfg->{pool};
> +    my $res = [];
> +    if (my $dat = $cache->{zfs}->{$zfspool}) {
> +
> +	foreach my $image (keys %$dat) {
> +
> +	    my $volname = $dat->{$image}->{name};
> +	    my $parent = $dat->{$image}->{parent};
> +	    my $owner = $dat->{$volname}->{vmid};
> +
> +	    next if ($volname =~ "state");
> +
> +	    my $volid = "$storeid:$volname";
> +
> +	    if ($vollist) {
> +		my $found = grep { $_ eq $volid } @$vollist;
> +		next if !$found;
> +	    } else {
> +		next if defined ($vmid) && ($owner ne $vmid);
> +	    }
> +
> +	    my $info = $dat->{$volname};
> +	    $info->{volid} = $volid;
> +	    push @$res, $info;
> +	}
> +    }
> +    return $res;
> +}
> +
> +sub status {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    my $total = 0;
> +    my $free = 0;
> +    my $used = 0;
> +    my $active = 0;
> +
> +    eval {
> +	($free, $used) = zfs_get_pool_stats($scfg);
> +	$active = 1;
> +	$total = $free + $used;
> +    };
> +    warn $@ if $@;
> +
> +    return ($total, $free, $used, $active);
> +}
> +
> +sub activate_storage {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +    return 1;
> +}
> +
> +sub deactivate_storage {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +    return 1;
> +}
> +
> +sub activate_volume {
> +    my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
> +    return 1;
> +}
> +
> +sub deactivate_volume {
> +    my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
> +    return 1;
> +}
> +
> +sub volume_size_info {
> +    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
> +    my $res = '';
> +    eval {
> +	$res = zfs_get_zvol_size($scfg, $volname);
> +    };
> +    warn $@ if $@;
> +    return $res;
> +}
> +
> +sub volume_resize {
> +    my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
> +
> +    my $new_size = ($size/1024);
> +
> +    eval {
> +	zfs_request($scfg, undef, 'set', 'volsize=' . $new_size . 'k', "$scfg->{pool}/$volname");
> +    };
> +    warn $@ if $@;
> +}
> +
> +sub volume_snapshot {
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
> +    eval {
> +	zfs_request($scfg, undef, 'snapshot', "$scfg->{pool}/$volname\@$snap");
> +    };
> +    warn $@ if $@;
> +}
> +
> +sub volume_snapshot_rollback {
> +    my ($class, $scfg, $storeid, $volname, $snap) = @_;
> +
> +    # abort rollback if snapshot is not the latest
> +    my @params = ('-t', 'snapshot', '-o', 'name', '-s', 'creation');
> +    my $text = zfs_request($scfg, undef, 'list', @params);
> +    my @snapshots = split(/\n/, $text);
> +    my $recentsnap = undef;
> +    foreach (@snapshots) {
> +        if (/$scfg->{pool}\/$volname/) {
> +            s/^.*@//;
> +            $recentsnap = $_;
> +        }
> +    }
> +    if ($snap ne $recentsnap) {
> +        die "cannot rollback, more recent snapshots exist\n";
> +    }
> +    eval {
> +	zfs_request($scfg, undef, 'rollback', "$scfg->{pool}/$volname\@$snap");
> +    };
> +    warn $@ if $@;
> +}
> +
> +sub volume_snapshot_delete {
> +    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
> +    eval {
> +	zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$volname\@$snap");
> +    };
> +    warn $@ if $@;
> +}
> +
> +sub volume_has_feature {
> +    my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
> +
> +    my $features = {
> +    snapshot => { current => 1, snap => 1},
> +    clone => { base => 1},
> +    template => { current => 1},
> +    copy => { base => 1, current => 1},
> +    };
> +
> +    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
> +    $class->parse_volname($volname);
> +
> +    my $key = undef;
> +
> +    if ($snapname) {
> +    $key = 'snap';
> +    } else {
> +    $key = $isBase ? 'base' : 'current';
> +    }
> +
> +    return 1 if $features->{$feature}->{$key};
> +
> +    return undef;
> +}
> +
> +1;
> -- 
> 1.7.10.4
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel at pve.proxmox.com
> http://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel



More information about the pve-devel mailing list