[pve-devel] [PATCH 1/5] This patch refactors ZFS's LUN management code, removing the existing LunCmds implementations, in favor of a single lun management infraestructure which just invokes a 'management helper' (which can be either an script or a native binary) which is invoked for lun creation/deletion/etc.
Pablo Ruiz García
pablo.ruiz at gmail.com
Sun Apr 13 02:08:10 CEST 2014
The 'protocol' which proxmox uses to communicate with the helper is simply
based on exposing a set of environment variables along with a few command
line arguments which are passed to helper script. This is based on the
same concept around CGI, AGI, and other similar decoupled invocation protocols.
Also, the helper can be invoked either locally or remotelly thru ssh (just
the same way it goes for invoking zfs itself), and some 'commands' expect
some kind of output as response (as it happends with Comstar's code). And
of course, helper's exit code is honored, so a non-zero exit code is
considered an error.
The actual commands supported by the 'helper protocol' are:
* create-lu
- Arguments: full_zvol_device_path
- Description: This command should handle creation of a LUN entry for the
given ZVOL device, and it is expected to return the LUN's
unique-id as output.
* resize-lu
- Arguments: size uuid_of_lu
- Description: This command should handle resizing of an specified LU's
attached ZVOL device.
* delete-lu
- Arguments: uuid_of_lu
- Description: This command should delete the specified LUN entry.
* share-lu
- Arguments: full_zvol_device_path
- Description: This command should (if not already done by 'create-lun')
expose/publish the LUN attached to the specified ZVOL.
Or do nothing if creation and sharing of LUN are both
tasks executed by 'create-lun'.
* get-lun-id
- Arguments: uuid_of_lu
- Description: This command should return the LUN number of the given
LUN unique-id as output.
* get-lun-no
- Arguments: uuid_of_lu
- Description: This command should return the given ZVOL/LUN's number.
This actual commands are modelled after the ones invoked by Comstar's LunCmd
perl module, which even if it maybe a bit too coupled to Comstar's/Solaris'
way of doing things. It seems to be flexible enough as to be usefull for other
implementations (as there are already one for Iet and one for BSD's Istgt).
Signed-off-by: Pablo Ruiz García <pablo.ruiz at gmail.com>
---
PVE/Storage/ZFSPlugin.pm | 170 ++++++++++++++++++++++++++++++---------------
1 files changed, 113 insertions(+), 57 deletions(-)
diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
index 4a8e9e4..80fc0ea 100644
--- a/PVE/Storage/ZFSPlugin.pm
+++ b/PVE/Storage/ZFSPlugin.pm
@@ -8,42 +8,95 @@ use PVE::Tools qw(run_command);
use PVE::Storage::Plugin;
use base qw(PVE::Storage::Plugin);
-use PVE::Storage::LunCmd::Comstar;
-use PVE::Storage::LunCmd::Istgt;
-use PVE::Storage::LunCmd::Iet;
my @ssh_opts = ('-o', 'BatchMode=yes');
my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
my $id_rsa_path = '/etc/pve/priv/zfs';
my $lun_cmds = {
- create_lu => 1,
- delete_lu => 1,
- import_lu => 1,
- modify_lu => 1,
- add_view => 1,
- list_view => 1,
- list_lu => 1,
+ 'create-lu' => 1, # Creates an iSCSI LUN for an existing ZFS zvol device.
+ 'delete-lu' => 1, # Removes an existing iSCSI LUN for a given ZFS vol device.
+ 'import-lu' => 1, # Imports a previously removed zvol device as iSCSI LUN.
+ 'resize-lu' => 1, # Signal iSCSI stack when a shared zvol has been resized.
+ 'share-lu' => 1, # Enables sharing of a zvol/iSCSI LUN resource.
+ 'get-lu-id' => 1, # Resolves a LUN's unique-id from it's device path.
+ 'get-lu-no' => 1, # Resolves a LUN's number, from it's assigned unique id.
};
-my $zfs_unknown_scsi_provider = sub {
- my ($provider) = @_;
+sub encode_cfg_value {
+ my ($key, $value) = @_;
- die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]";
-};
+ if ($key eq 'nodes' || $key eq 'content') {
+ return join(',', keys(%$value));
+ }
-my $zfs_get_base = sub {
- my ($scfg) = @_;
+ return $value;
+}
+
+sub run_lun_command {
+ my ($scfg, $timeout, $method, @params) = @_;
- if ($scfg->{iscsiprovider} eq 'comstar') {
- return PVE::Storage::LunCmd::Comstar::get_base;
- } elsif ($scfg->{iscsiprovider} eq 'istgt') {
- return PVE::Storage::LunCmd::Istgt::get_base;
- } elsif ($scfg->{iscsiprovider} eq 'iet') {
- return PVE::Storage::LunCmd::Iet::get_base;
+ my $msg = '';
+ my %vars = ();
+ my ($guid, $env, $lundev, $size) = undef;
+
+ my $helper = "$scfg->{lunhelper} $method ";
+ die "No 'lunhelper' defined" if !$helper;
+
+ $timeout = 10 if !$timeout;
+
+ if ($method eq 'create-lu') {
+ $lundev = $params[0];
+ $helper .= "$lundev ";
+ } elsif ($method eq 'delete-lu') {
+ $guid = $params[0];
+ $helper .= "$guid";
+ } elsif ($method eq 'resize-lu') {
+ $size = $params[0];
+ $guid = $params[1];
+ $helper .= "$guid $size";
+ } elsif ($method eq 'share-lu') {
+ $lundev = $params[0];
+ $helper .= "$lundev";
+ } elsif ($method eq 'get-lu-id') {
+ $lundev = $params[0];
+ $helper .= "$lundev";
+ } elsif ($method eq 'get-lu-no') {
+ $guid = $params[0];
+ $helper .= "$guid";
} else {
- $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
+ die "$method not implemented yet!";
}
+
+ # Common environment variables
+ $vars{SSHKEY} = "$id_rsa_path/$scfg->{portal}_id_rsa";
+ $vars{LUNDEV} = $lundev if $lundev;
+ $vars{LUNUUID} = $guid if $guid;
+
+ foreach my $k (keys %$scfg) {
+ $env .= "PMXCFG_$k=\"". encode_cfg_value($k, $scfg->{$k}) ."\" ";
+ }
+ foreach my $k (keys %vars) {
+ $env .= "PMXVAR_$k=\"$vars{$k}\" ";
+ }
+
+ my $output = sub {
+ my $line = shift;
+ $msg .= "$line";
+ };
+
+ my $target = 'root@' . $scfg->{portal};
+ my $cmd = !$scfg->{remotehelper} ? "$env $helper"
+ : [@ssh_cmd, '-i'. "$id_rsa_path/$scfg->{portal}_id_rsa", $target, "$env $helper"];
+
+ run_command($cmd, timeout => $timeout, outfunc => $output);
+
+ return $msg;
+}
+
+my $zfs_get_base = sub {
+ my ($scfg) = @_;
+ return $scfg->{devbase};
};
sub zfs_request {
@@ -57,15 +110,7 @@ sub zfs_request {
$timeout = 5 if !$timeout;
if ($lun_cmds->{$method}) {
- if ($scfg->{iscsiprovider} eq 'comstar') {
- $msg = PVE::Storage::LunCmd::Comstar::run_lun_command($scfg, $timeout, $method, @params);
- } elsif ($scfg->{iscsiprovider} eq 'istgt') {
- $msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params);
- } elsif ($scfg->{iscsiprovider} eq 'iet') {
- $msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params);
- } else {
- $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
- }
+ $msg = run_lun_command($scfg, $timeout, $method, @params);
} else {
if ($method eq 'zpool_list') {
$zfscmd = 'zpool';
@@ -175,7 +220,7 @@ sub zfs_parse_zvol_list {
return $list;
}
-sub zfs_get_lu_name {
+sub zfs_get_lu_guid {
my ($scfg, $zvol) = @_;
my $object;
@@ -186,11 +231,11 @@ sub zfs_get_lu_name {
$object = "$base/$scfg->{pool}/$zvol";
}
- my $lu_name = zfs_request($scfg, undef, 'list_lu', $object);
+ my $guid = zfs_request($scfg, undef, 'get-lu-id', $object);
- return $lu_name if $lu_name;
+ return $guid if $guid;
- die "Could not find lu_name for zvol $zvol";
+ die "Could not find guid for zvol $zvol";
}
sub zfs_get_zvol_size {
@@ -205,29 +250,29 @@ sub zfs_get_zvol_size {
die "Could not get zvol size";
}
-sub zfs_add_lun_mapping_entry {
+sub zfs_share_lu {
my ($scfg, $zvol, $guid) = @_;
if (! defined($guid)) {
- $guid = zfs_get_lu_name($scfg, $zvol);
+ $guid = zfs_get_lu_guid($scfg, $zvol);
}
- zfs_request($scfg, undef, 'add_view', $guid);
+ zfs_request($scfg, undef, 'share-lu', $guid);
}
sub zfs_delete_lu {
my ($scfg, $zvol) = @_;
- my $guid = zfs_get_lu_name($scfg, $zvol);
+ my $guid = zfs_get_lu_guid($scfg, $zvol);
- zfs_request($scfg, undef, 'delete_lu', $guid);
+ zfs_request($scfg, undef, 'delete-lu', $guid);
}
sub zfs_create_lu {
my ($scfg, $zvol) = @_;
my $base = $zfs_get_base->($scfg);
- my $guid = zfs_request($scfg, undef, 'create_lu', "$base/$scfg->{pool}/$zvol");
+ my $guid = zfs_request($scfg, undef, 'create-lu', "$base/$scfg->{pool}/$zvol");
return $guid;
}
@@ -236,15 +281,15 @@ sub zfs_import_lu {
my ($scfg, $zvol) = @_;
my $base = $zfs_get_base->($scfg);
- zfs_request($scfg, undef, 'import_lu', "$base/$scfg->{pool}/$zvol");
+ zfs_request($scfg, undef, 'import-lu', "$base/$scfg->{pool}/$zvol");
}
sub zfs_resize_lu {
my ($scfg, $zvol, $size) = @_;
- my $guid = zfs_get_lu_name($scfg, $zvol);
+ my $guid = zfs_get_lu_guid($scfg, $zvol);
- zfs_request($scfg, undef, 'modify_lu', "${size}K", $guid);
+ zfs_request($scfg, undef, 'resize-lu', "${size}K", $guid);
}
sub zfs_create_zvol {
@@ -264,7 +309,7 @@ sub zfs_get_lun_number {
die "could not find lun_number for guid $guid" if !$guid;
- return zfs_request($scfg, undef, 'list_view', $guid);
+ return zfs_request($scfg, undef, 'get-lu-no', $guid);
}
sub zfs_list_zvol {
@@ -315,10 +360,19 @@ sub plugindata {
sub properties {
return {
- iscsiprovider => {
- description => "iscsi provider",
- type => 'string',
- },
+ devbase => {
+ description => "ZFS device's base path at remote host (ie. /dev)",
+ type => 'string',
+ },
+ lunhelper => {
+ description => "Helper command used by ZFS plugin handle lun creation/deletion/etc.",
+ type => 'string',
+ },
+ remotehelper => {
+ description => "Wether helper command should be invoked locally (at pmx host) or remotelly (at ZFS server).",
+ type => 'boolean',
+ optional => 1,
+ },
blocksize => {
description => "block size",
type => 'string',
@@ -334,8 +388,10 @@ sub options {
target => { fixed => 1 },
pool => { fixed => 1 },
blocksize => { fixed => 1 },
- iscsiprovider => { fixed => 1 },
content => { optional => 1 },
+ lunhelper => { fixed => 1 },
+ remotehelper => { optional => 1 },
+ devbase => { fixed => 1 }
};
}
@@ -359,7 +415,7 @@ sub path {
my $target = $scfg->{target};
my $portal = $scfg->{portal};
- my $guid = zfs_get_lu_name($scfg, $name);
+ my $guid = zfs_get_lu_guid($scfg, $name);
my $lun = zfs_get_lun_number($scfg, $guid);
my $path = "iscsi://$portal/$target/$lun";
@@ -411,7 +467,7 @@ sub create_base {
zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
my $guid = zfs_create_lu($scfg, $newname);
- zfs_add_lun_mapping_entry($scfg, $newname, $guid);
+ zfs_share_lu($scfg, $newname, $guid);
my $running = undef; #fixme : is create_base always offline ?
@@ -437,7 +493,7 @@ sub clone_image {
zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
my $guid = zfs_create_lu($scfg, $name);
- zfs_add_lun_mapping_entry($scfg, $name, $guid);
+ zfs_share_lu($scfg, $name, $guid);
return $name;
}
@@ -454,7 +510,7 @@ sub alloc_image {
zfs_create_zvol($scfg, $name, $size);
my $guid = zfs_create_lu($scfg, $name);
- zfs_add_lun_mapping_entry($scfg, $name, $guid);
+ zfs_share_lu($scfg, $name, $guid);
return $name;
}
@@ -471,7 +527,7 @@ sub free_image {
do {
my $err = $@;
my $guid = zfs_create_lu($scfg, $name);
- zfs_add_lun_mapping_entry($scfg, $name, $guid);
+ zfs_share_lu($scfg, $name, $guid);
die $err;
} if $@;
@@ -585,7 +641,7 @@ sub volume_snapshot_rollback {
zfs_import_lu($scfg, $volname);
- zfs_add_lun_mapping_entry($scfg, $volname);
+ zfs_share_lu($scfg, $volname);
}
sub volume_snapshot_delete {
--
1.7.1
More information about the pve-devel
mailing list