[pve-devel] [PATCH 01/09] Added support for ZFS Storage Plugin

mir at datanom.net mir at datanom.net
Mon Aug 12 19:55:27 CEST 2013


From: Michael Rasmussen <mir at datanom.net>

Signed-off-by: Michael Rasmussen <mir at datanom.net>
---
 PVE/Storage.pm           |   2 +
 PVE/Storage/Makefile     |   2 +-
 PVE/Storage/Plugin.pm    |   2 +-
 PVE/Storage/ZFSPlugin.pm | 646 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 650 insertions(+), 2 deletions(-)
 create mode 100644 PVE/Storage/ZFSPlugin.pm

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index c6d9d12..8dbd46f 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -25,6 +25,7 @@ use PVE::Storage::RBDPlugin;
 use PVE::Storage::SheepdogPlugin;
 use PVE::Storage::ISCSIDirectPlugin;
 use PVE::Storage::NexentaPlugin;
+use PVE::Storage::ZFSPlugin;
 
 # load and initialize all plugins
 PVE::Storage::DirPlugin->register();
@@ -35,6 +36,7 @@ PVE::Storage::RBDPlugin->register();
 PVE::Storage::SheepdogPlugin->register();
 PVE::Storage::ISCSIDirectPlugin->register();
 PVE::Storage::NexentaPlugin->register();
+PVE::Storage::ZFSPlugin->register();
 PVE::Storage::Plugin->init();
 
 my $UDEVADM = '/sbin/udevadm';
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index ed65a18..88b6161 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm ZFSPlugin.pm
 
 .PHONY: install
 install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 88e34a9..323ff5d 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -312,7 +312,7 @@ sub parse_config {
 	    $d->{content} = $def->{content}->[1] if !$d->{content};
 	}
 
-	if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' ) {
+	if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' || $type eq 'zfs') {
 	    $d->{shared} = 1;
 	}
     }
diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
new file mode 100644
index 0000000..909f71f
--- /dev/null
+++ b/PVE/Storage/ZFSPlugin.pm
@@ -0,0 +1,646 @@
+package PVE::Storage::ZFSPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use POSIX;
+use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
+use PVE::Storage::Plugin;
+use PVE::JSONSchema qw(get_standard_option);
+
+use base qw(PVE::Storage::Plugin);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+
+sub zfs_request {
+    my ($scfg, $method, @params) = @_;
+	my $luncmd = ();
+	my $cmd;
+	my $msg = '';
+	my $zfs;
+	
+	my $output = sub {
+		my $line = shift;
+		$msg .= "$line\n";
+	};
+	
+	if (uc($scfg->{iscsiprovider}) eq 'COMSTAR') {
+		$luncmd = {
+			cmd		=> '/usr/sbin/stmfadm',
+			list_lu 	=> 'list-lu',
+			create_lu	=> 'create-lu',
+			delete_lu	=> 'delete-lu',
+			import_lu	=> 'import-lu',
+			add_view	=> 'add-view',
+			list_view	=> 'list-view',
+		};
+	} else {
+		die 'unknown iscsi provider. Available [comstar]';
+	}
+	
+	$luncmd->{cmd} = 'sudo ' . $luncmd->{cmd} if ($scfg->{sudo});
+	if ($luncmd->{$method}) {
+		$cmd = [@ssh_cmd, $scfg->{portal}, $luncmd->{cmd}, $luncmd->{$method}, @params];
+	} else {
+		if ($method =~ /^zpool-list$/) {
+			$zfs = 'zpool';
+			$method = 'list';
+		} else {
+			$zfs = 'zfs';
+		}
+		$cmd = [@ssh_cmd, $scfg->{portal}, $zfs, $method, @params];
+	}
+	
+	eval {
+		run_command($cmd, outfunc => $output, errmsg => 'zfs error');
+	};
+	if ($@) {
+		my $cmdstr = '';
+		foreach my $c (@$cmd) {
+			if ($cmdstr eq '') {
+				$cmdstr = $c;
+			} else {
+				$cmdstr .= " $c";
+			}
+		}
+		die $@ . " [$cmdstr]";
+	}
+	
+	return $msg;
+}
+
+sub zfs_parse_size {
+    my ($text) = @_;
+
+    return 0 if !$text;
+
+    if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
+	my ($size, $reminder, $unit) = ($1, $2, $3);
+	return $size if !$unit;
+	if ($unit eq 'K') {
+	    $size *= 1024;
+	} elsif ($unit eq 'M') {
+	    $size *= 1024*1024;
+	} elsif ($unit eq 'G') {
+	    $size *= 1024*1024*1024;
+	} elsif ($unit eq 'T') {
+	    $size *= 1024*1024*1024*1024;
+	}
+		if ($reminder) {
+			$size = ceil($size);
+		}
+		return $size;
+    } else {
+		return 0;
+    }
+}
+
+sub zfs_get_zvol_props {
+    my ($scfg, $zvol) = @_;
+    my $props = {};
+
+    my $text = zfs_request($scfg, 'get', '-H', 'all', $zvol);
+
+    my @lines = split /\n/, $text;
+    foreach my $line (@lines) {
+	if ($line =~ /^(.+)\s+(.+)\s+(.+)\s+(.+)$/) {
+            $props->{$2} = $3;
+        }
+    }
+    
+    return $props;
+}
+
+sub zfs_get_pool_stats {
+	my ($scfg) = @_;
+	my $map = {
+		size => 0,
+		used => 0,
+	};
+	my $text;
+	
+	$text = zfs_request($scfg, 'zpool-list', '-H', '-o',
+	                       'size', $scfg->{pool});
+	$map->{size} = zfs_parse_size($text);
+
+	$text = zfs_request($scfg, 'get', '-o', 'value', '-Hp',
+	                       'used', $scfg->{pool});
+	$map->{used} = $text;
+	
+	return $map;
+}
+
+sub zfs_parse_zvol_list {
+    my ($text) = @_;
+    my @list = ();
+    my @zvols;
+
+    return \@list if !$text;
+
+    my @lines = split /\n/, $text;
+    foreach my $line (@lines) {
+		@zvols = split /\//, $line;
+		if (scalar(@zvols) == 2 && $zvols[0] !~ /^rpool$/) {
+			push (@list, $zvols[0] . '/' . $zvols[1]);
+		}
+    }
+
+    return \@list;
+}
+
+sub zfs_parse_lu_list {
+    my ($text) = @_;
+    my @list = ();
+    my $lun;
+
+    return undef if !$text || $text eq '';
+
+    my @lines = split /\n/, $text;
+
+    foreach my $line (@lines) {
+        if ($line =~ /^\s*LU Name\s*:\s*([a-zA-Z0-9]+)$/) {
+			$lun = ();
+			$lun->{lun} = $1;
+        }
+        elsif ($line =~ /^\s*Data File\s*:\s*([a-zA-Z0-9\-\/]+)$/) {
+			$lun->{source} = $1;
+        }
+        elsif ($line =~ /^\s*Size\s*:\s*([a-zA-Z0-9]+)$/) {
+			$lun->{size} = $1;
+			push (@list, $lun);
+        }
+    }
+
+    return \@list;
+}
+
+sub zfs_get_lu_info {
+    my ($list, $source) = @_;
+    my @list = ();
+
+    return \@list if (!$source || !$list);
+    
+    foreach my $lun (@$list) {
+    	push (@list, $lun) if $lun->{source} =~ /^$source$/;
+    }
+    
+    return \@list;
+}
+ 
+sub zfs_list_lun_mapping_entries {
+    my ($scfg, $zvol) = @_;
+    my $object;
+
+    my $text = zfs_request($scfg, 'list_lu', '-v');
+    my $list = zfs_parse_lu_list($text);
+	return undef if (! defined($list));
+  
+    if ($zvol =~ /^.+\/.+/) {
+        $object = "/dev/zvol/rdsk/$zvol";
+    }
+    else {
+        $object = "/dev/zvol/rdsk/$scfg->{pool}/$zvol";
+    }
+
+    return zfs_get_lu_info($list, $object);
+}
+
+sub zfs_get_zvol_size {
+    my ($scfg, $zvol) = @_;
+
+    my $text = zfs_request($scfg, 'get', '-H', 'volsize', "$scfg->{pool}/$zvol");
+    $text =~ /^.+\s+\w+\s+(\d+\w)\s+.+$/;
+   
+    return zfs_parse_size($1);
+}
+
+sub zfs_add_lun_mapping_entry {
+    my ($scfg, $zvol) = @_;
+    
+    my $luns = zfs_list_lun_mapping_entries($scfg, $zvol);
+	die "Could not find any LUN's" if (! defined($luns));
+
+    zfs_request($scfg, 'add_view', @$luns[0]->{lun});
+}
+
+sub zfs_delete_lu {
+    my ($scfg, $zvol) = @_;
+
+    my $luns = zfs_list_lun_mapping_entries($scfg, $zvol);
+	die "Could not find any LUN's" if (! defined($luns));
+
+    zfs_request($scfg, 'delete_lu', @$luns[0]->{lun});
+}
+
+sub zfs_create_lu {
+    my ($scfg, $zvol) = @_;
+
+    zfs_request($scfg, 'create_lu', "/dev/zvol/rdsk/$scfg->{pool}/$zvol");
+}
+
+sub zfs_import_lu {
+    my ($scfg, $zvol) = @_;
+
+    zfs_request($scfg, 'import_lu', "/dev/zvol/rdsk/$scfg->{pool}/$zvol");
+}
+
+sub zfs_create_zvol {
+    my ($scfg, $zvol, $size) = @_;
+
+    zfs_request($scfg, 'create', '-b', $scfg->{bsize}, '-V',
+                "${size}k", "$scfg->{pool}/$zvol");
+}
+
+sub zfs_delete_zvol {
+    my ($scfg, $zvol) = @_;
+
+    zfs_request($scfg, 'destroy', '-r', "$scfg->{pool}/$zvol");
+}
+
+sub zfs_list_zvol {
+    my ($scfg) = @_;
+
+    my $text = zfs_request($scfg, 'list', '-o', 'name', '-Hr');
+    my $zvols = zfs_parse_zvol_list($text);
+    return undef if !$zvols;
+
+    my $list = {};
+    foreach my $zvol (@$zvols) {
+		my @values = split('/', $zvol);
+
+		my $pool = $values[0];
+		my $image = $values[1];
+		my $owner;
+		if ($image =~ m/^((vm|base)-(\d+)-\S+)$/) {
+			$owner = $3;
+		}
+
+		my $props = zfs_get_zvol_props($scfg, $zvol);
+		my $parent = $props->{origin};
+		if($parent && $parent =~ m/^$scfg->{pool}\/(\S+)$/){
+			$parent = $1;
+		}
+
+		$list->{$pool}->{$image} = {
+			name => $image,
+			size => zfs_get_zvol_size($scfg, $image),
+			format => 'raw',
+			vmid => $owner
+		};
+    }
+
+    return $list;
+}
+
+sub zfs_get_lun_number {
+	my ($scfg, $name) = @_;
+	my $lunnum = undef;
+	
+	my $text = zfs_request($scfg, 'list_view', '-l', $name);
+	my @lines = split /\n/, $text;
+
+    foreach my $line (@lines) {
+        if ($line =~ /^\s*LUN\s*:\s*(\d+)$/) {
+			$lunnum = $1;
+			last;
+		}
+	}
+	
+	return $lunnum;
+}
+
+# Configuration
+
+sub type {
+    return 'zfs';
+}
+
+sub plugindata {
+    return {
+		content => [ {images => 1}, { images => 1 }],
+    };
+}
+
+sub properties {
+    return {
+		chap => {
+			description => "chap",
+			type => 'string',
+		},
+		pwd => {
+			description => "password",
+			type => 'string',
+		},
+		bsize => {
+			description => "block size",
+			type => 'string',
+		},
+		iscsiprovider => {
+			description => "iscsi provider",
+			type => 'string',
+		},
+		sudo => {
+			description => "use sudo",
+			type => 'boolean',
+		},
+    };
+}
+
+sub options {
+    return {
+        nodes => { optional => 1 },
+        disable => { optional => 1 },
+        portal => { fixed => 1 },
+		target => { fixed => 1 },
+        pool => { fixed => 1 },
+		chap => { optional => 1 },
+		pwd => { optional => 1 },
+        bsize => { fixed => 1 },
+		iscsiprovider => { fixed => 1 },
+		sudo => { optional => 1 },
+		content => { optional => 1 },
+    };
+}
+
+# Storage implementation
+
+sub parse_volname {
+    my ($class, $volname) = @_;
+
+    if ($volname =~ m/^(((base|vm)-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
+		return ('images', $5, $8, $2, $4, $6);
+	}
+
+    die "unable to parse zfs volume name '$volname'\n";
+}
+
+sub path {
+    my ($class, $scfg, $volname) = @_;
+
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+    
+    my $target = $scfg->{target};
+    my $portal = $scfg->{portal};
+
+    my $map = zfs_list_lun_mapping_entries($scfg, $name);
+    die "could not find lun number" if !$map;
+    
+    my $lun = zfs_get_lun_number($scfg, @$map[0]->{lun});
+    die "lun is not OK\n" if (! defined($lun));   
+    my $path = "iscsi://$portal/$target/$lun";
+
+    return ($path, $vmid, $vtype);
+}
+
+my $find_free_diskname = sub {
+    my ($storeid, $scfg, $vmid) = @_;
+
+    my $name = undef;
+    my $volumes = zfs_list_zvol($scfg);
+	die "unable de get zvol list" if !$volumes;
+
+    my $disk_ids = {};
+    my $dat = $volumes->{$scfg->{pool}};
+
+    foreach my $image (keys %$dat) {
+        my $volname = $dat->{$image}->{name};
+        if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
+            $disk_ids->{$2} = 1;
+        }
+    }
+
+    #fix: can we search in $rbd hash key with a regex to find (vm|base) ?
+    for (my $i = 1; $i < 100; $i++) {
+        if (!$disk_ids->{$i}) {
+            return "vm-$vmid-disk-$i";
+        }
+    }
+
+    die "unable to allocate an image name for VM $vmid in storage '$storeid'\n"
+
+};
+
+sub create_base {
+    my ($class, $storeid, $scfg, $volname) = @_;
+
+    my $snap = '__base__';
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+        $class->parse_volname($volname);
+
+    die "create_base not possible with base image\n" if $isBase;
+
+    my $newname = $name;
+    $newname =~ s/^vm-/base-/;
+
+    my $newvolname = $basename ? "$basename/$newname" : "$newname";
+
+    zfs_delete_lu($scfg, $name);
+    zfs_request($scfg, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
+
+    zfs_create_lu($scfg, $newname);
+    zfs_add_lun_mapping_entry($scfg, $newname);
+
+    my $running  = undef; #fixme : is create_base always offline ?
+
+    $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
+
+    return $newvolname;
+}
+
+sub clone_image {
+    my ($class, $scfg, $storeid, $volname, $vmid) = @_;
+
+    my $snap = '__base__';
+
+    my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
+        $class->parse_volname($volname);
+
+    die "clone_image only works on base images\n" if !$isBase;
+
+    my $name = &$find_free_diskname($storeid, $scfg, $vmid);
+
+    warn "clone $volname: $basename to $name\n";
+
+    zfs_request($scfg, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
+
+    zfs_create_lu($scfg, $name);
+    zfs_add_lun_mapping_entry($scfg, $name);
+
+    return $name;
+}
+
+sub alloc_image {
+    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+
+    die "unsupported format '$fmt'" if $fmt ne 'raw';
+
+    die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
+	if $name && $name !~ m/^vm-$vmid-/;
+
+	$name = &$find_free_diskname($storeid, $scfg, $vmid);
+    die "unable to allocate an image name for VM $vmid in storage '$storeid'\n"
+	if !$name;
+
+    zfs_create_zvol($scfg, $name, $size);
+    zfs_create_lu($scfg, $name);
+    zfs_add_lun_mapping_entry($scfg, $name);
+
+    return $name;
+}
+
+sub free_image {
+    my ($class, $storeid, $scfg, $volname, $isBase) = @_;
+
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+	eval {
+    	zfs_delete_lu($scfg, $name);
+	};
+   	zfs_delete_zvol($scfg, $name);
+
+    return undef;
+}
+
+sub list_images {
+    my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+
+    $cache->{zfs} = zfs_list_zvol($scfg) if !$cache->{zfs};
+    my $zfspool = $scfg->{pool};
+    my $res = [];
+
+    if (my $dat = $cache->{zfs}->{$zfspool}) {
+		foreach my $image (keys %$dat) {
+	
+			my $volname = $dat->{$image}->{name};
+			my $parent = $dat->{$image}->{parent};
+			
+			my $volid = undef;
+            if ($parent && $parent =~ m/^(\S+)@(\S+)$/) {
+                my ($basename) = ($1);
+                $volid = "$storeid:$basename/$volname";
+            } else {
+                $volid = "$storeid:$volname";
+            }
+			my $owner = $dat->{$volname}->{vmid};
+			if ($vollist) {
+				my $found = grep { $_ eq $volid } @$vollist;
+				next if !$found;
+			} else {
+				next if defined ($vmid) && ($owner ne $vmid);
+			}
+
+			my $info = $dat->{$volname};
+			$info->{volid} = $volid;
+
+			push @$res, $info;
+	
+		}
+    }
+
+    return $res;
+}
+
+sub status {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    my $total = 0;
+    my $free = 0;
+    my $used = 0;
+    my $active = 0;
+
+    eval {
+		my $map = zfs_get_pool_stats($scfg);
+		$active = 1;
+		$total = zfs_parse_size($map->{size});
+		$used = zfs_parse_size($map->{used});
+		$free = $total - $used;
+    };
+    warn $@ if $@;
+
+    return ($total, $free, $used, $active);
+}
+
+sub activate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+    return 1;
+}
+
+sub deactivate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+    return 1;
+}
+
+sub activate_volume {
+    my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+    return 1;
+}
+
+sub deactivate_volume {
+    my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+    return 1;
+}
+
+sub volume_size_info {
+    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+    return zfs_get_zvol_size($scfg, $volname),
+}
+
+sub volume_resize {
+    my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
+
+    zfs_request($scfg, 'set', 'volsize=' . ($size/1024) . 'k', "$scfg->{pool}/$volname");
+}
+
+sub volume_snapshot {
+    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+    zfs_request($scfg, 'snapshot', "$scfg->{pool}/$volname\@$snap");
+}
+
+sub volume_snapshot_rollback {
+    my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+    zfs_delete_lu($scfg, $volname);
+
+    zfs_request($scfg, 'rollback', "$scfg->{pool}/$volname\@$snap");
+    
+    zfs_import_lu($scfg, $volname);
+    
+    zfs_add_lun_mapping_entry($scfg, $volname);
+}
+
+sub volume_snapshot_delete {
+    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+    zfs_request($scfg, 'destroy', "$scfg->{pool}/$volname\@$snap");
+}
+
+sub volume_has_feature {
+    my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+    my $features = {
+		snapshot => { current => 1, snap => 1},
+		clone => { base => 1},
+		template => { current => 1},
+		copy => { base => 1, current => 1},
+    };
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+	$class->parse_volname($volname);
+
+    my $key = undef;
+    if($snapname){
+		$key = 'snap';
+    } else {
+		$key =  $isBase ? 'base' : 'current';
+    }
+    return 1 if $features->{$feature}->{$key};
+
+    return undef;
+}
+
+1;
+
-- 
1.8.4.rc2




More information about the pve-devel mailing list