[pve-devel] [PATCH 1/8] Added support for ZFS Storage Plugin
mir at datanom.net
mir at datanom.net
Thu Jun 27 22:28:03 CEST 2013
From: Michael Rasmussen <mir at datanom.net>
Signed-off-by: Michael Rasmussen <mir at datanom.net>
---
PVE/Storage.pm | 2 +
PVE/Storage/Makefile | 2 +-
PVE/Storage/Plugin.pm | 2 +-
PVE/Storage/ZFSPlugin.pm | 646 +++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 650 insertions(+), 2 deletions(-)
create mode 100644 PVE/Storage/ZFSPlugin.pm
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index c6d9d12..8dbd46f 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -25,6 +25,7 @@ use PVE::Storage::RBDPlugin;
use PVE::Storage::SheepdogPlugin;
use PVE::Storage::ISCSIDirectPlugin;
use PVE::Storage::NexentaPlugin;
+use PVE::Storage::ZFSPlugin;
# load and initialize all plugins
PVE::Storage::DirPlugin->register();
@@ -35,6 +36,7 @@ PVE::Storage::RBDPlugin->register();
PVE::Storage::SheepdogPlugin->register();
PVE::Storage::ISCSIDirectPlugin->register();
PVE::Storage::NexentaPlugin->register();
+PVE::Storage::ZFSPlugin->register();
PVE::Storage::Plugin->init();
my $UDEVADM = '/sbin/udevadm';
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index ed65a18..88b6161 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm ZFSPlugin.pm
.PHONY: install
install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 88e34a9..323ff5d 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -312,7 +312,7 @@ sub parse_config {
$d->{content} = $def->{content}->[1] if !$d->{content};
}
- if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' ) {
+ if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' || $type eq 'zfs') {
$d->{shared} = 1;
}
}
diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
new file mode 100644
index 0000000..909f71f
--- /dev/null
+++ b/PVE/Storage/ZFSPlugin.pm
@@ -0,0 +1,646 @@
+package PVE::Storage::ZFSPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use POSIX;
+use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
+use PVE::Storage::Plugin;
+use PVE::JSONSchema qw(get_standard_option);
+
+use base qw(PVE::Storage::Plugin);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+
+sub zfs_request {
+ my ($scfg, $method, @params) = @_;
+ my $luncmd = ();
+ my $cmd;
+ my $msg = '';
+ my $zfs;
+
+ my $output = sub {
+ my $line = shift;
+ $msg .= "$line\n";
+ };
+
+ if (uc($scfg->{iscsiprovider}) eq 'COMSTAR') {
+ $luncmd = {
+ cmd => '/usr/sbin/stmfadm',
+ list_lu => 'list-lu',
+ create_lu => 'create-lu',
+ delete_lu => 'delete-lu',
+ import_lu => 'import-lu',
+ add_view => 'add-view',
+ list_view => 'list-view',
+ };
+ } else {
+ die 'unknown iscsi provider. Available [comstar]';
+ }
+
+ $luncmd->{cmd} = 'sudo ' . $luncmd->{cmd} if ($scfg->{sudo});
+ if ($luncmd->{$method}) {
+ $cmd = [@ssh_cmd, $scfg->{portal}, $luncmd->{cmd}, $luncmd->{$method}, @params];
+ } else {
+ if ($method =~ /^zpool-list$/) {
+ $zfs = 'zpool';
+ $method = 'list';
+ } else {
+ $zfs = 'zfs';
+ }
+ $cmd = [@ssh_cmd, $scfg->{portal}, $zfs, $method, @params];
+ }
+
+ eval {
+ run_command($cmd, outfunc => $output, errmsg => 'zfs error');
+ };
+ if ($@) {
+ my $cmdstr = '';
+ foreach my $c (@$cmd) {
+ if ($cmdstr eq '') {
+ $cmdstr = $c;
+ } else {
+ $cmdstr .= " $c";
+ }
+ }
+ die $@ . " [$cmdstr]";
+ }
+
+ return $msg;
+}
+
+sub zfs_parse_size {
+ my ($text) = @_;
+
+ return 0 if !$text;
+
+ if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
+ my ($size, $reminder, $unit) = ($1, $2, $3);
+ return $size if !$unit;
+ if ($unit eq 'K') {
+ $size *= 1024;
+ } elsif ($unit eq 'M') {
+ $size *= 1024*1024;
+ } elsif ($unit eq 'G') {
+ $size *= 1024*1024*1024;
+ } elsif ($unit eq 'T') {
+ $size *= 1024*1024*1024*1024;
+ }
+ if ($reminder) {
+ $size = ceil($size);
+ }
+ return $size;
+ } else {
+ return 0;
+ }
+}
+
+sub zfs_get_zvol_props {
+ my ($scfg, $zvol) = @_;
+ my $props = {};
+
+ my $text = zfs_request($scfg, 'get', '-H', 'all', $zvol);
+
+ my @lines = split /\n/, $text;
+ foreach my $line (@lines) {
+ if ($line =~ /^(.+)\s+(.+)\s+(.+)\s+(.+)$/) {
+ $props->{$2} = $3;
+ }
+ }
+
+ return $props;
+}
+
+sub zfs_get_pool_stats {
+ my ($scfg) = @_;
+ my $map = {
+ size => 0,
+ used => 0,
+ };
+ my $text;
+
+ $text = zfs_request($scfg, 'zpool-list', '-H', '-o',
+ 'size', $scfg->{pool});
+ $map->{size} = zfs_parse_size($text);
+
+ $text = zfs_request($scfg, 'get', '-o', 'value', '-Hp',
+ 'used', $scfg->{pool});
+ $map->{used} = $text;
+
+ return $map;
+}
+
+sub zfs_parse_zvol_list {
+ my ($text) = @_;
+ my @list = ();
+ my @zvols;
+
+ return \@list if !$text;
+
+ my @lines = split /\n/, $text;
+ foreach my $line (@lines) {
+ @zvols = split /\//, $line;
+ if (scalar(@zvols) == 2 && $zvols[0] !~ /^rpool$/) {
+ push (@list, $zvols[0] . '/' . $zvols[1]);
+ }
+ }
+
+ return \@list;
+}
+
+sub zfs_parse_lu_list {
+ my ($text) = @_;
+ my @list = ();
+ my $lun;
+
+ return undef if !$text || $text eq '';
+
+ my @lines = split /\n/, $text;
+
+ foreach my $line (@lines) {
+ if ($line =~ /^\s*LU Name\s*:\s*([a-zA-Z0-9]+)$/) {
+ $lun = ();
+ $lun->{lun} = $1;
+ }
+ elsif ($line =~ /^\s*Data File\s*:\s*([a-zA-Z0-9\-\/]+)$/) {
+ $lun->{source} = $1;
+ }
+ elsif ($line =~ /^\s*Size\s*:\s*([a-zA-Z0-9]+)$/) {
+ $lun->{size} = $1;
+ push (@list, $lun);
+ }
+ }
+
+ return \@list;
+}
+
+sub zfs_get_lu_info {
+ my ($list, $source) = @_;
+ my @list = ();
+
+ return \@list if (!$source || !$list);
+
+ foreach my $lun (@$list) {
+ push (@list, $lun) if $lun->{source} =~ /^$source$/;
+ }
+
+ return \@list;
+}
+
+sub zfs_list_lun_mapping_entries {
+ my ($scfg, $zvol) = @_;
+ my $object;
+
+ my $text = zfs_request($scfg, 'list_lu', '-v');
+ my $list = zfs_parse_lu_list($text);
+ return undef if (! defined($list));
+
+ if ($zvol =~ /^.+\/.+/) {
+ $object = "/dev/zvol/rdsk/$zvol";
+ }
+ else {
+ $object = "/dev/zvol/rdsk/$scfg->{pool}/$zvol";
+ }
+
+ return zfs_get_lu_info($list, $object);
+}
+
+sub zfs_get_zvol_size {
+ my ($scfg, $zvol) = @_;
+
+ my $text = zfs_request($scfg, 'get', '-H', 'volsize', "$scfg->{pool}/$zvol");
+ $text =~ /^.+\s+\w+\s+(\d+\w)\s+.+$/;
+
+ return zfs_parse_size($1);
+}
+
+sub zfs_add_lun_mapping_entry {
+ my ($scfg, $zvol) = @_;
+
+ my $luns = zfs_list_lun_mapping_entries($scfg, $zvol);
+ die "Could not find any LUN's" if (! defined($luns));
+
+ zfs_request($scfg, 'add_view', @$luns[0]->{lun});
+}
+
+sub zfs_delete_lu {
+ my ($scfg, $zvol) = @_;
+
+ my $luns = zfs_list_lun_mapping_entries($scfg, $zvol);
+ die "Could not find any LUN's" if (! defined($luns));
+
+ zfs_request($scfg, 'delete_lu', @$luns[0]->{lun});
+}
+
+sub zfs_create_lu {
+ my ($scfg, $zvol) = @_;
+
+ zfs_request($scfg, 'create_lu', "/dev/zvol/rdsk/$scfg->{pool}/$zvol");
+}
+
+sub zfs_import_lu {
+ my ($scfg, $zvol) = @_;
+
+ zfs_request($scfg, 'import_lu', "/dev/zvol/rdsk/$scfg->{pool}/$zvol");
+}
+
+sub zfs_create_zvol {
+ my ($scfg, $zvol, $size) = @_;
+
+ zfs_request($scfg, 'create', '-b', $scfg->{bsize}, '-V',
+ "${size}k", "$scfg->{pool}/$zvol");
+}
+
+sub zfs_delete_zvol {
+ my ($scfg, $zvol) = @_;
+
+ zfs_request($scfg, 'destroy', '-r', "$scfg->{pool}/$zvol");
+}
+
+sub zfs_list_zvol {
+ my ($scfg) = @_;
+
+ my $text = zfs_request($scfg, 'list', '-o', 'name', '-Hr');
+ my $zvols = zfs_parse_zvol_list($text);
+ return undef if !$zvols;
+
+ my $list = {};
+ foreach my $zvol (@$zvols) {
+ my @values = split('/', $zvol);
+
+ my $pool = $values[0];
+ my $image = $values[1];
+ my $owner;
+ if ($image =~ m/^((vm|base)-(\d+)-\S+)$/) {
+ $owner = $3;
+ }
+
+ my $props = zfs_get_zvol_props($scfg, $zvol);
+ my $parent = $props->{origin};
+ if($parent && $parent =~ m/^$scfg->{pool}\/(\S+)$/){
+ $parent = $1;
+ }
+
+ $list->{$pool}->{$image} = {
+ name => $image,
+ size => zfs_get_zvol_size($scfg, $image),
+ format => 'raw',
+ vmid => $owner
+ };
+ }
+
+ return $list;
+}
+
+sub zfs_get_lun_number {
+ my ($scfg, $name) = @_;
+ my $lunnum = undef;
+
+ my $text = zfs_request($scfg, 'list_view', '-l', $name);
+ my @lines = split /\n/, $text;
+
+ foreach my $line (@lines) {
+ if ($line =~ /^\s*LUN\s*:\s*(\d+)$/) {
+ $lunnum = $1;
+ last;
+ }
+ }
+
+ return $lunnum;
+}
+
+# Configuration
+
+sub type {
+ return 'zfs';
+}
+
+sub plugindata {
+ return {
+ content => [ {images => 1}, { images => 1 }],
+ };
+}
+
+sub properties {
+ return {
+ chap => {
+ description => "chap",
+ type => 'string',
+ },
+ pwd => {
+ description => "password",
+ type => 'string',
+ },
+ bsize => {
+ description => "block size",
+ type => 'string',
+ },
+ iscsiprovider => {
+ description => "iscsi provider",
+ type => 'string',
+ },
+ sudo => {
+ description => "use sudo",
+ type => 'boolean',
+ },
+ };
+}
+
+sub options {
+ return {
+ nodes => { optional => 1 },
+ disable => { optional => 1 },
+ portal => { fixed => 1 },
+ target => { fixed => 1 },
+ pool => { fixed => 1 },
+ chap => { optional => 1 },
+ pwd => { optional => 1 },
+ bsize => { fixed => 1 },
+ iscsiprovider => { fixed => 1 },
+ sudo => { optional => 1 },
+ content => { optional => 1 },
+ };
+}
+
+# Storage implementation
+
+sub parse_volname {
+ my ($class, $volname) = @_;
+
+ if ($volname =~ m/^(((base|vm)-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
+ return ('images', $5, $8, $2, $4, $6);
+ }
+
+ die "unable to parse zfs volume name '$volname'\n";
+}
+
+sub path {
+ my ($class, $scfg, $volname) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ my $target = $scfg->{target};
+ my $portal = $scfg->{portal};
+
+ my $map = zfs_list_lun_mapping_entries($scfg, $name);
+ die "could not find lun number" if !$map;
+
+ my $lun = zfs_get_lun_number($scfg, @$map[0]->{lun});
+ die "lun is not OK\n" if (! defined($lun));
+ my $path = "iscsi://$portal/$target/$lun";
+
+ return ($path, $vmid, $vtype);
+}
+
+my $find_free_diskname = sub {
+ my ($storeid, $scfg, $vmid) = @_;
+
+ my $name = undef;
+ my $volumes = zfs_list_zvol($scfg);
+ die "unable de get zvol list" if !$volumes;
+
+ my $disk_ids = {};
+ my $dat = $volumes->{$scfg->{pool}};
+
+ foreach my $image (keys %$dat) {
+ my $volname = $dat->{$image}->{name};
+ if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
+ $disk_ids->{$2} = 1;
+ }
+ }
+
+ #fix: can we search in $rbd hash key with a regex to find (vm|base) ?
+ for (my $i = 1; $i < 100; $i++) {
+ if (!$disk_ids->{$i}) {
+ return "vm-$vmid-disk-$i";
+ }
+ }
+
+ die "unable to allocate an image name for VM $vmid in storage '$storeid'\n"
+
+};
+
+sub create_base {
+ my ($class, $storeid, $scfg, $volname) = @_;
+
+ my $snap = '__base__';
+
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+ $class->parse_volname($volname);
+
+ die "create_base not possible with base image\n" if $isBase;
+
+ my $newname = $name;
+ $newname =~ s/^vm-/base-/;
+
+ my $newvolname = $basename ? "$basename/$newname" : "$newname";
+
+ zfs_delete_lu($scfg, $name);
+ zfs_request($scfg, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
+
+ zfs_create_lu($scfg, $newname);
+ zfs_add_lun_mapping_entry($scfg, $newname);
+
+ my $running = undef; #fixme : is create_base always offline ?
+
+ $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
+
+ return $newvolname;
+}
+
+sub clone_image {
+ my ($class, $scfg, $storeid, $volname, $vmid) = @_;
+
+ my $snap = '__base__';
+
+ my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
+ $class->parse_volname($volname);
+
+ die "clone_image only works on base images\n" if !$isBase;
+
+ my $name = &$find_free_diskname($storeid, $scfg, $vmid);
+
+ warn "clone $volname: $basename to $name\n";
+
+ zfs_request($scfg, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
+
+ zfs_create_lu($scfg, $name);
+ zfs_add_lun_mapping_entry($scfg, $name);
+
+ return $name;
+}
+
+sub alloc_image {
+ my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+
+ die "unsupported format '$fmt'" if $fmt ne 'raw';
+
+ die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
+ if $name && $name !~ m/^vm-$vmid-/;
+
+ $name = &$find_free_diskname($storeid, $scfg, $vmid);
+ die "unable to allocate an image name for VM $vmid in storage '$storeid'\n"
+ if !$name;
+
+ zfs_create_zvol($scfg, $name, $size);
+ zfs_create_lu($scfg, $name);
+ zfs_add_lun_mapping_entry($scfg, $name);
+
+ return $name;
+}
+
+sub free_image {
+ my ($class, $storeid, $scfg, $volname, $isBase) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ eval {
+ zfs_delete_lu($scfg, $name);
+ };
+ zfs_delete_zvol($scfg, $name);
+
+ return undef;
+}
+
+sub list_images {
+ my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+
+ $cache->{zfs} = zfs_list_zvol($scfg) if !$cache->{zfs};
+ my $zfspool = $scfg->{pool};
+ my $res = [];
+
+ if (my $dat = $cache->{zfs}->{$zfspool}) {
+ foreach my $image (keys %$dat) {
+
+ my $volname = $dat->{$image}->{name};
+ my $parent = $dat->{$image}->{parent};
+
+ my $volid = undef;
+ if ($parent && $parent =~ m/^(\S+)@(\S+)$/) {
+ my ($basename) = ($1);
+ $volid = "$storeid:$basename/$volname";
+ } else {
+ $volid = "$storeid:$volname";
+ }
+ my $owner = $dat->{$volname}->{vmid};
+ if ($vollist) {
+ my $found = grep { $_ eq $volid } @$vollist;
+ next if !$found;
+ } else {
+ next if defined ($vmid) && ($owner ne $vmid);
+ }
+
+ my $info = $dat->{$volname};
+ $info->{volid} = $volid;
+
+ push @$res, $info;
+
+ }
+ }
+
+ return $res;
+}
+
+sub status {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ my $total = 0;
+ my $free = 0;
+ my $used = 0;
+ my $active = 0;
+
+ eval {
+ my $map = zfs_get_pool_stats($scfg);
+ $active = 1;
+ $total = zfs_parse_size($map->{size});
+ $used = zfs_parse_size($map->{used});
+ $free = $total - $used;
+ };
+ warn $@ if $@;
+
+ return ($total, $free, $used, $active);
+}
+
+sub activate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ return 1;
+}
+
+sub deactivate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ return 1;
+}
+
+sub activate_volume {
+ my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+ return 1;
+}
+
+sub deactivate_volume {
+ my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+ return 1;
+}
+
+sub volume_size_info {
+ my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+ return zfs_get_zvol_size($scfg, $volname),
+}
+
+sub volume_resize {
+ my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
+
+ zfs_request($scfg, 'set', 'volsize=' . ($size/1024) . 'k', "$scfg->{pool}/$volname");
+}
+
+sub volume_snapshot {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ zfs_request($scfg, 'snapshot', "$scfg->{pool}/$volname\@$snap");
+}
+
+sub volume_snapshot_rollback {
+ my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+ zfs_delete_lu($scfg, $volname);
+
+ zfs_request($scfg, 'rollback', "$scfg->{pool}/$volname\@$snap");
+
+ zfs_import_lu($scfg, $volname);
+
+ zfs_add_lun_mapping_entry($scfg, $volname);
+}
+
+sub volume_snapshot_delete {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ zfs_request($scfg, 'destroy', "$scfg->{pool}/$volname\@$snap");
+}
+
+sub volume_has_feature {
+ my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+ my $features = {
+ snapshot => { current => 1, snap => 1},
+ clone => { base => 1},
+ template => { current => 1},
+ copy => { base => 1, current => 1},
+ };
+
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+ $class->parse_volname($volname);
+
+ my $key = undef;
+ if($snapname){
+ $key = 'snap';
+ } else {
+ $key = $isBase ? 'base' : 'current';
+ }
+ return 1 if $features->{$feature}->{$key};
+
+ return undef;
+}
+
+1;
+
--
1.8.3.1
More information about the pve-devel
mailing list