[pve-devel] [PATCH storage 1/2] add SCST provider to ZFS over iSCSI storage type

Dzmitry Kotsikau dkotsikau at gmail.com
Thu May 3 14:50:21 CEST 2018


Signed-off-by: Dzmitry Kotsikau <dkotsikau at gmail.com>
---
 PVE/Storage/LunCmd/Makefile |   2 +-
 PVE/Storage/LunCmd/Scst.pm  | 557 ++++++++++++++++++++++++++++++++++++++++++++
 PVE/Storage/ZFSPlugin.pm    |  15 +-
 3 files changed, 570 insertions(+), 4 deletions(-)
 create mode 100644 PVE/Storage/LunCmd/Scst.pm

diff --git a/PVE/Storage/LunCmd/Makefile b/PVE/Storage/LunCmd/Makefile
index b959255..5a44cbd 100644
--- a/PVE/Storage/LunCmd/Makefile
+++ b/PVE/Storage/LunCmd/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Comstar.pm Istgt.pm Iet.pm
+SOURCES=Comstar.pm Istgt.pm Iet.pm Scst.pm
 
 .PHONY: install
 install:
diff --git a/PVE/Storage/LunCmd/Scst.pm b/PVE/Storage/LunCmd/Scst.pm
new file mode 100644
index 0000000..4e0a67b
--- /dev/null
+++ b/PVE/Storage/LunCmd/Scst.pm
@@ -0,0 +1,557 @@
+package PVE::Storage::LunCmd::Scst;
+
+#1) Install scst on proxmox node
+
+#apt-get install git gawk build-essential flex bison automake autoconf pve-headers dkms
+#git clone https://github.com/bvanassche/scst.git
+#cd scst
+#make scst scst_local iscsi-scst scst srpt usr
+#make scst_install scst_local_install iscsi_install scst_install srpt_install  usr_install
+#depmod -a
+#cat > /etc/default/scst << 'EOF'
+#ISCSID_OPTIONS="-a 192.168.254.10 -u0 -g0 -p3260"
+#SCST_TARGET_MODULES="scst scst_vdisk iscsi-scst"
+#EOF
+
+#2) Create portal
+
+#cat > /etc/scst.conf <'EOF'
+#TARGET_DRIVER iscsi {
+#        DefaultTime2Wait 2
+#        DefaultTime2Retain 90
+#        enabled 1
+#}
+#EOF
+
+
+
+
+use strict;
+use warnings;
+use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
+use Data::Dumper;
+
+sub get_base;
+
+# A logical unit can max have 16383 LUNs
+my $MAX_LUNS = 16383;
+my $HANDLER = 'vdisk_fileio';
+
+my $CONFIG_FILE = '/etc/scst.conf';
+my $CONFIG_FILE_TMP = '/tmp/pve-scst.conf';
+
+my $DAEMON = '/usr/local/sbin/iscsi-scstd';
+my $SETTINGS = undef;
+my $CONFIG;
+
+my @ssh_opts = ('-o', 'BatchMode=yes', '-o','PreferredAuthentications=publickey');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my @scp_cmd = ('/usr/bin/scp', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+my $scstadmin = '/usr/local/sbin/scstadmin';
+
+my $execute_command = sub {
+    my ($scfg, $exec, $timeout, $method, @params) = @_;
+
+    my $msg = '';
+    my $err = undef;
+    my $target;
+    my $cmd;
+    my $res = ();
+
+    $timeout = 10 if !$timeout;
+
+    my $output = sub {
+    my $line = shift;
+    $msg .= "$line\n";
+    };
+
+    my $errfunc = sub {
+    my $line = shift;
+    $err .= "$line";
+    };
+
+    if ($exec eq 'scp') {
+        $target = 'root@[' . $scfg->{portal} . ']';
+        $cmd = [@scp_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", '--', $method, "$target:$params[0]"];
+    } else {
+        $target = 'root@' . $scfg->{portal};
+        $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $method, @params];
+    }
+
+    eval {
+        run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout);
+    };
+    if ($@) {
+        $res = {
+            result => 0,
+            msg => $err,
+        }
+    } else {
+        $res = {
+            result => 1,
+            msg => $msg,
+        }
+    }
+
+    return $res;
+};
+
+my $update_config = sub {
+    my ($scfg) = @_;
+
+    my @params = ('-write_config ', $CONFIG_FILE);
+    my $res = $execute_command->($scfg, 'ssh', undef, $scstadmin, @params);
+    die $res->{msg} unless $res->{result};
+};
+
+my $get_target_tid = sub {
+    my ($scfg) = @_;
+
+    my $res = {msg => undef };
+
+    my @params = ("/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/tid");
+    $res = $execute_command->($scfg, 'ssh', 10, 'test', '-f',  @params);
+    die "The target not found! ". $res->{msg} unless $res->{result};
+
+    my $tid = undef;
+
+    $res = $execute_command->($scfg, 'ssh', undef, 'cat', @params);
+    die $res->{msg} unless $res->{result};
+    my @lines = split "\n", $res->{msg};
+    $tid =  $lines[0];
+    return $tid;
+};
+
+
+sub parseLine {
+    my $line = shift;
+    my $hash = shift;
+    my $child = shift;
+    return if (! $line);
+    return if ($line =~ /^\s*$/);
+
+    $line =~ s/^\s+//; $line =~ s/\s+$//;
+
+    my @elements;
+    while ($line =~ m/"([^"\\]*(\\.[^"\\]*)*)"|([^\s]+)/g) {
+        push @elements, defined($1) ? $1:$3;
+    }
+
+    my $attribute = $elements[0];
+    my $value     = $elements[1];
+    my $value2    = $elements[2];
+
+    if (defined($attribute) && defined($value) && defined($value2)) {
+        $$hash{$attribute}->{$value}->{$value2} = $child;
+    } elsif (defined($attribute) && defined($value)) {
+        $$hash{$attribute}->{$value} = $child;
+    } elsif (defined($attribute)) {
+        $$hash{$attribute} = $child;
+    }
+};
+
+
+sub parseStanza {
+    my $buffer = shift;
+    my $line;
+    my %hash;
+    my $attribute;
+    my $value;
+    my $value2;
+    my $quoted;
+
+    while ($#{$buffer} > -1) {
+            my $char = shift @{$buffer};
+
+            if ($char eq '{') {
+                my $child = parseStanza($buffer);
+                    if ($line) {
+                            parseLine($line, \%hash, $child);
+                            $line = undef;
+                        }
+                    next;
+            }
+
+             if ($char eq '}') {
+                return \%hash
+            };
+
+            if ($char eq "\n") {
+                    my %empty;
+                    parseLine($line, \%hash, \%empty);
+                    $line = undef;
+            } else {
+                    $line .= $char;
+            }
+    }
+    return \%hash;
+};
+
+sub parseLuns {
+    my $scfg = shift;
+    my $luns = shift;
+    my $h_luns = ();
+    my $base = get_base;
+
+    foreach my $lun (keys %{$$luns} ) {
+        foreach my $device (keys %{$$luns->{$lun}}) {
+                if (!defined($$CONFIG{'HANDLER'}) ||
+                    !defined($$CONFIG{'HANDLER'}->{$HANDLER}) ||
+                    !defined($$CONFIG{'HANDLER'}->{$HANDLER}->{'DEVICE'}->{$device})) {
+                        die "Wrong configuration file $CONFIG_FILE!";
+                }
+                my $dev = $$CONFIG{'HANDLER'}->{$HANDLER}->{'DEVICE'}->{$device};
+
+                my $conf = undef;
+                $conf->{Device}    = (keys %{$$dev{'prod_id'}})[0];
+                $conf->{Path}      = (keys %{$$dev{'filename'}})[0];
+                $conf->{blocksize} = (keys %{$$dev{'blocksize'}})[0];
+                $conf->{size}      = (keys %{$$dev{'size'}})[0];
+                if ($conf->{Path} && $conf->{Path} =~ /^$base\/$scfg->{pool}\/([\w\-]+)$/) {
+                    $conf->{include} = 1;
+                } else {
+                    $conf->{include} = 0;
+                }
+                $conf->{lun} = $lun;
+                push @{$h_luns}, $conf;
+            }
+    }
+    return $h_luns;
+}
+
+my $readConfigFile = sub {
+    my ($scfg) = @_;
+    my $buffer;
+    my @stanza;
+    my $level;
+    my $used = ();
+
+    my @commands = ("touch '$CONFIG_FILE_TMP'", 
+                    "chmod 600 '$CONFIG_FILE_TMP'",
+                    "$scstadmin -write_config '$CONFIG_FILE_TMP' -nonkey",
+                    "cat '$CONFIG_FILE_TMP'",
+                    "rm -f '$CONFIG_FILE_TMP'"
+    );
+    my $res = $execute_command->($scfg, 'ssh', 20, join('&&', @commands));
+    die $res->{msg} unless $res->{result};
+    my @lines = split "\n", $res->{msg};
+    foreach my $line (@lines){
+        $line =~ s/^\#.*//;
+        $line =~ s/[^\\]\#.*//;
+        $line =~ s/\\(.)/$1/g;
+        $buffer .= $line."\n";
+    }
+    my @buff_a;
+    @buff_a = split(//, $buffer);
+    $CONFIG = parseStanza(\@buff_a);
+
+    if (!defined($$CONFIG{'TARGET_DRIVER'}) ||
+            !(scalar keys %{$$CONFIG{'TARGET_DRIVER'}}) ||
+            !defined($$CONFIG{'TARGET_DRIVER'}->{'iscsi'}) ||
+            !defined($$CONFIG{'TARGET_DRIVER'}->{'iscsi'}->{'TARGET'}->{$scfg->{target}})
+    )
+    {
+        die  Dumper($CONFIG) ."\nWrong configuration file $CONFIG_FILE!";
+
+    }
+
+    $SETTINGS->{target} = $scfg->{target};
+    my $tgt = $$CONFIG{'TARGET_DRIVER'}->{'iscsi'}->{'TARGET'}->{$scfg->{target}};
+    if (defined($$tgt{'LUN'})) {
+	push @{$SETTINGS->{luns}}, @{parseLuns($scfg ,\$tgt->{'LUN'})};
+    }
+
+    if (defined($$tgt{'GROUP'})) {
+        foreach my $group (keys %{$$tgt{'GROUP'}}) {
+	    if (defined($$tgt{'GROUP'}->{$group}->{'LUN'})) {
+		push @{$SETTINGS->{luns}}, @{parseLuns($scfg , \$tgt->{'GROUP'}->{$group}->{'LUN'} )};
+	    }
+	}
+    }
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+	$used->{$lun->{lun}} = 1;
+    }
+
+    $SETTINGS->{used} = $used;
+#    die Dumper($SETTINGS);
+    return 0;
+};
+
+my $get_lu_name = sub {
+    my $i;
+
+    my $used = $SETTINGS->{used};
+    for ($i = 0; $i < $MAX_LUNS; $i++) {
+        last unless $used->{$i};
+    }
+    $SETTINGS->{used}->{$i} = 1;
+
+    return $i;
+};
+
+my $init_lu_name = sub {
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        $SETTINGS->{used}->{$lun->{lun}} = 1;
+    }
+};
+
+my $free_lu_name = sub {
+    my ($lu_name) = @_;
+    my $new;
+
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        if ($lun->{lun} != $lu_name) {
+            push @$new, $lun;
+        }
+    }
+
+    $SETTINGS->{luns} = $new;
+    $SETTINGS->{used}->{$lu_name} = 0;
+};
+
+my $make_lun = sub {
+    my ($scfg, $path) = @_;
+
+    die 'Maximum number of LUNs per target is 16383' if scalar @{$SETTINGS->{luns}} >= $MAX_LUNS;
+
+    my $lun = $get_lu_name->();
+    my $conf = {
+        lun => $lun,
+        Path => $path,
+        include => 1,
+    };
+    push @{$SETTINGS->{luns}}, $conf;
+    $SETTINGS->{used}->{$lun} = 1;
+    # print Dumper ($SETTINGS);
+    return $conf;
+};
+
+my $list_view = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $lun = undef;
+
+    my $object = $params[0];
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        next unless $lun->{include} == 1;
+        if ($lun->{Path} =~ /^$object$/) {
+            return $lun->{lun} if (defined($lun->{lun}));
+            die "$lun->{Path}: Missing LUN";
+        }
+    }
+
+    return $lun;
+};
+
+my $list_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $name = undef;
+
+    my $object = $params[0];
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        next unless $lun->{include} == 1;
+        if ($lun->{Path} =~ /^$object$/) {
+            return $lun->{Path};
+        }
+    }
+    return $name;
+};
+
+my $parse_size = sub {
+    my ($text) = @_;
+
+    return 0 if !$text;
+
+    if ($text =~ m/^(\d+(\.\d+)?)(k)?$/) {
+    my ($size, $reminder, $unit) = ($1, $2, $3);
+    return $size if !$unit;
+    if ($unit eq 'k') {
+        $size *= 1024;
+    }
+        if ($reminder) {
+            $size = ceil($size);
+        }
+        return $size;
+    } else {
+        return 0;
+    }
+};
+
+sub append_group_params {
+    my $scfg = shift;
+    my $params = shift;
+    if ($scfg->{comstar_tg}) {
+	push @{$params}, ('-group', $scfg->{comstar_tg} );
+    }
+}
+
+my $create_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+
+    if ($list_lun->($scfg, $timeout, $method, @params)) {
+        die "$params[0]: LUN exists";
+    }
+    my $lun = $params[0];
+    $lun = $make_lun->($scfg, $lun);
+
+    my $tid = $get_target_tid->($scfg);
+    my $device = 'diskt'.$tid.'l'.$lun->{lun};
+    print "Create $HANDLER device $device using filename=$lun->{Path}...";
+    my $blocksize = $parse_size->($scfg->{blocksize});
+    my $res = {msg => undef};
+
+
+    my @commands = (
+	"(i=1; while [ ! -e '$lun->{Path}' -a  \$i -le 20 ]; do sleep 1; i=\$((i+1)); done)",
+	"[ -e '$lun->{Path}' ]",
+        "$scstadmin");
+    @params = ('-open_dev', $device, '-handler', $HANDLER, '-attributes', "filename=$lun->{Path},nv_cache=1,blocksize=512,thin_provisioned=1,zero_copy=1");
+
+    $res = $execute_command->($scfg, 'ssh', 30, join ('&&',  @commands), @params);
+#    $res = $execute_command->($scfg, 'ssh', 30, $scstadmin, @params);
+    do {
+        $free_lu_name->($lun->{lun});
+	if ($res->{msg}) {
+	    die  $res->{msg} ;
+	} else {
+	    die 'create_lun: timeout or unknown error! command: '. join ('&&',  @commands) . ' ' . join ' ' , @params;
+	}
+    } unless $res->{result};
+
+    print "Done!\n";
+    print "Create LUN $lun->{lun} using the device $device for target $scfg->{target}...";
+
+    @params = ('-add_lun', $lun->{lun}, '-driver', 'iscsi', '-device', $device, '-target', "$scfg->{target}");
+    append_group_params ($scfg, \@params);
+
+    $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+
+    do {
+        print "Failed!\nRemoving device...";
+        @params = ('-close_dev', $device, '-handler', $HANDLER,'-noprompt');
+        $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+        print "Done!\n";
+        $free_lu_name->($lun->{lun});
+        $update_config->($scfg);
+        die $res->{msg};
+    } unless $res->{result};
+    print "Done!\n";
+    $update_config->($scfg);
+    return $res->{msg};
+};
+
+my $delete_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $res = {msg => undef};
+
+    my $path = $params[0];
+
+
+    foreach my $lun (@{$SETTINGS->{luns}}) {
+        if ($lun->{Path} eq $path) {
+
+            my $tid = $get_target_tid->($scfg);
+            my $device = 'diskt'.$tid.'l'.$lun->{lun};
+	    print "Delete $HANDLER device $device using filename=$lun->{Path}...";
+
+            @params = ('-rem_lun', $lun->{lun},  '-driver', 'iscsi', '-device', $device, '-target', "$scfg->{target}", '-noprompt');
+            append_group_params ($scfg, \@params);
+
+            $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+            do {
+                $free_lu_name->($lun->{lun});
+                die $res->{msg};
+            } unless $res->{result};
+
+	    print "Done!\n";
+	    print "Delete LUN $lun->{lun} using the device $device for target $scfg->{target}...";
+
+            @params = ('-close_dev', $device, '-handler', $HANDLER,'-noprompt');
+            $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+
+            if ($res->{result}) {
+                $free_lu_name->($lun->{lun});
+	        print "Done!\n";
+                last;
+            } else {
+                $update_config->($scfg);
+                die $res->{msg};
+            }
+        }
+    }
+    $update_config->($scfg);
+    return $res->{msg};
+};
+
+my $import_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+
+    print "Import LUN\n";
+    return $create_lun->($scfg, $timeout, $method, @params);
+};
+
+my $modify_lun = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    my $lun;
+    my $res = {msg => undef};
+
+    my $path = $params[1];
+
+    foreach my $cfg (@{$SETTINGS->{luns}}) {
+        if ($cfg->{Path} eq $path) {
+            $lun = $cfg;
+            last;
+        }
+    }
+
+    my $tid = $get_target_tid->($scfg);
+    my $device = 'diskt'.$tid.'l'.$lun->{lun};
+
+
+    @params = ('-resync_dev',  $device, '-handler', $HANDLER,'-noprompt');
+    $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+    die $res->{msg} unless $res->{result};
+
+    $update_config->($scfg);
+    return $res->{msg};
+};
+
+my $add_view = sub {
+    my ($scfg, $timeout, $method, @params) = @_;
+    return '';
+};
+
+my $get_lun_cmd_map = sub {
+    my ($method) = @_;
+
+    my $cmdmap = {
+        create_lu   =>  { cmd => $create_lun },
+        delete_lu   =>  { cmd => $delete_lun },
+        import_lu   =>  { cmd => $import_lun },
+        modify_lu   =>  { cmd => $modify_lun },
+        add_view    =>  { cmd => $add_view },
+        list_view   =>  { cmd => $list_view },
+        list_lu     =>  { cmd => $list_lun },
+    };
+
+    die "unknown command '$method'" unless exists $cmdmap->{$method};
+
+    return $cmdmap->{$method};
+};
+
+sub run_lun_command {
+    my ($scfg, $timeout, $method, @params) = @_;
+
+    $readConfigFile->($scfg) unless $SETTINGS;
+    my $cmdmap = $get_lun_cmd_map->($method);
+    my $msg = $cmdmap->{cmd}->($scfg, $timeout, $method, @params);
+
+    return $msg;
+}
+
+sub get_base {
+    return '/dev/zvol';
+}
+
+1;
+
diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
index f88fe94..6694d4e 100644
--- a/PVE/Storage/ZFSPlugin.pm
+++ b/PVE/Storage/ZFSPlugin.pm
@@ -12,9 +12,11 @@ use base qw(PVE::Storage::ZFSPoolPlugin);
 use PVE::Storage::LunCmd::Comstar;
 use PVE::Storage::LunCmd::Istgt;
 use PVE::Storage::LunCmd::Iet;
+use PVE::Storage::LunCmd::Scst;
+use Data::Dumper;
 
 
-my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_opts = ('-o', 'BatchMode=yes','-o','PreferredAuthentications=publickey');
 my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
 my $id_rsa_path = '/etc/pve/priv/zfs';
 
@@ -31,7 +33,7 @@ my $lun_cmds = {
 my $zfs_unknown_scsi_provider = sub {
     my ($provider) = @_;
 
-    die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]";
+    die "$provider: unknown iscsi provider. Available [comstar, istgt, iet, scst]";
 };
 
 my $zfs_get_base = sub {
@@ -43,6 +45,8 @@ my $zfs_get_base = sub {
         return PVE::Storage::LunCmd::Istgt::get_base;
     } elsif ($scfg->{iscsiprovider} eq 'iet') {
         return PVE::Storage::LunCmd::Iet::get_base;
+    } elsif ($scfg->{iscsiprovider} eq 'scst') {
+        return PVE::Storage::LunCmd::Scst::get_base;
     } else {
         $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
     }
@@ -63,6 +67,8 @@ sub zfs_request {
             $msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params);
         } elsif ($scfg->{iscsiprovider} eq 'iet') {
             $msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params);
+        } elsif ($scfg->{iscsiprovider} eq 'scst') {
+            $msg = PVE::Storage::LunCmd::Scst::run_lun_command($scfg, $timeout, $method, @params);
         } else {
             $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
         }
@@ -98,11 +104,13 @@ sub zfs_get_lu_name {
 
     my $object = ($zvol =~ /^.+\/.+/) ? "$base/$zvol" : "$base/$scfg->{pool}/$zvol";
 
+    print "$object\n";
+
     my $lu_name = $class->zfs_request($scfg, undef, 'list_lu', $object);
 
     return $lu_name if $lu_name;
 
-    die "Could not find lu_name for zvol $zvol";
+    die "Could not find lu_name for zvol $zvol: $object";
 }
 
 sub zfs_add_lun_mapping_entry {
@@ -351,6 +359,7 @@ sub volume_has_feature {
 	clone => { base => 1},
 	template => { current => 1},
 	copy => { base => 1, current => 1},
+	sparseinit => { base => 1, current => 1},
     };
 
     my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
-- 
2.11.0



More information about the pve-devel mailing list