[pve-devel] [PATCH pve-storage 1/5] First beta of FreeNAS storage plugin. Missing snapshot backup for LXC
Michael Rasmussen
mir at datanom.net
Fri Jun 16 03:10:28 CEST 2017
Signed-off-by: Michael Rasmussen <mir at datanom.net>
---
PVE/Storage.pm | 2 +
PVE/Storage/FreeNASPlugin.pm | 1289 ++++++++++++++++++++++++++++++++++++++++++
PVE/Storage/Makefile | 2 +-
PVE/Storage/Plugin.pm | 2 +-
4 files changed, 1293 insertions(+), 2 deletions(-)
create mode 100644 PVE/Storage/FreeNASPlugin.pm
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index ee2295a..f33249f 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -32,6 +32,7 @@ use PVE::Storage::GlusterfsPlugin;
use PVE::Storage::ZFSPoolPlugin;
use PVE::Storage::ZFSPlugin;
use PVE::Storage::DRBDPlugin;
+use PVE::Storage::FreeNASPlugin;
# Storage API version. Icrement it on changes in storage API interface.
use constant APIVER => 1;
@@ -49,6 +50,7 @@ PVE::Storage::GlusterfsPlugin->register();
PVE::Storage::ZFSPoolPlugin->register();
PVE::Storage::ZFSPlugin->register();
PVE::Storage::DRBDPlugin->register();
+PVE::Storage::FreeNASPlugin->register();
# load third-party plugins
if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) {
diff --git a/PVE/Storage/FreeNASPlugin.pm b/PVE/Storage/FreeNASPlugin.pm
new file mode 100644
index 0000000..771061e
--- /dev/null
+++ b/PVE/Storage/FreeNASPlugin.pm
@@ -0,0 +1,1289 @@
+package PVE::Storage::FreeNASPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use POSIX;
+use PVE::Tools qw(run_command);
+use PVE::Storage::Plugin;
+use PVE::RPCEnvironment;
+use PVE::QemuServer;
+use PVE::LXC;
+use LWP::UserAgent;
+use HTTP::Status;
+use JSON;
+use Data::Dumper;
+
+use base qw(PVE::Storage::Plugin);
+
+my $api = '/api/v1.0';
+my $timeout = 60; # seconds
+my $max_luns = 20; # maximum supported luns per target group in freenas is 25
+ # but reserve 5 for temporary LUNs (snapshots with RAM and
+ # snapshot backup)
+my $active_snaps = 4;
+my $limit = 10000; # limit for get requests
+my $version;
+my $fullversion;
+my $target_prefix = 'iqn.2005-10.org.freenas.ctl';
+
+sub freenas_request {
+ my ($scfg, $request, $section, $data, $valid_code) = @_;
+ my $ua = LWP::UserAgent->new;
+ $ua->agent("ProxmoxUA/0.1");
+ $ua->ssl_opts( verify_hostname => 0 );
+ $ua->timeout($timeout);
+ push @{ $ua->requests_redirectable }, 'POST';
+ push @{ $ua->requests_redirectable }, 'PUT';
+ push @{ $ua->requests_redirectable }, 'DELETE';
+ my $req;
+
+ my $url = "https://$scfg->{portal}$api/$section";
+
+ if ($request eq 'GET') {
+ $req = HTTP::Request->new(GET => $url);
+ } elsif ($request eq 'POST') {
+ $req = HTTP::Request->new(POST => $url);
+ $req->content($data);
+ } elsif ($request eq 'PUT') {
+ $req = HTTP::Request->new(PUT => $url);
+ $req->content($data);
+ } elsif ($request eq 'DELETE') {
+ $req = HTTP::Request->new(DELETE => $url);
+ } else {
+ die "$request: Unknown request";
+ }
+
+ $req->content_type('application/json');
+ $req->authorization_basic($scfg->{username}, $scfg->{password});
+ my $res = $ua->request($req);
+
+ #print Dumper($res);
+ return $res->code unless $res->is_success;
+
+ return $res->content;
+}
+
+sub freenas_get_version {
+ my ($scfg) = @_;
+
+ my $response = freenas_request($scfg, 'GET', "system/version");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $info = decode_json($response);
+ $fullversion = $info->{fullversion};
+ if ($fullversion =~ /^\w+-(\d+)\.(\d*)\.(\d*)/) {
+ my $minor = $2;
+ my $micro = $3;
+
+ if ($minor) {
+ $minor = "0$minor" unless $minor > 9;
+ } else {
+ $minor = '00';
+ }
+
+ if ($micro) {
+ $micro = "0$micro" unless $micro > 9;
+ } else {
+ $micro = '00';
+ }
+
+ $version = "$1$minor$micro";
+ } else {
+ $version = '90200';
+ }
+}
+
+sub freenas_list_zvol {
+ my ($scfg) = @_;
+
+ freenas_get_version($scfg) unless $version;
+
+ my $response = freenas_request($scfg, 'GET', "storage/volume/$scfg->{pool}/zvols?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $zvols = decode_json($response);
+ $response = freenas_request($scfg, 'GET', "storage/snapshot?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $snapshots = decode_json($response);
+
+ my $list = ();
+ my $hide = {};
+ my $vmid;
+ my $parent;
+ foreach my $zvol (@$zvols) {
+ next unless $zvol->{name} =~ /^(base|vm)-(\d+)-disk-\d+$/;
+ $vmid = $2;
+ $parent = undef;
+ foreach my $snap (@$snapshots) {
+ next unless $snap->{name} eq "__base__$vmid";
+ $parent = $snap->{filesystem} =~ /^$scfg->{pool}\/(.+)$/ ? $1 : undef;
+ }
+ $list->{$scfg->{pool}}->{$zvol->{name}} = {
+ name => $zvol->{name},
+ size => $zvol->{volsize},
+ parent => $parent,
+ vmid => $vmid,
+ format => 'raw',
+ };
+ if ($zvol->{name} =~ /^base-(.*)/) {
+ $hide->{"vm-$1"} = 1;
+ }
+ }
+
+ delete @{$list->{$scfg->{pool}}}{keys %$hide};
+
+ return $list;
+}
+
+sub freenas_no_more_extents {
+ my ($scfg, $target) = @_;
+
+ my $response = freenas_request($scfg, 'GET', "services/iscsi/targettoextent?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $extents = decode_json($response);
+ foreach my $extent (@$extents) {
+ return 0 if $extent->{iscsi_target} == $target;
+ }
+
+ return 1;
+}
+
+sub freenas_get_target {
+ my ($scfg, $vmid) = @_;
+ my $target = undef;
+
+ my $response = freenas_request($scfg, 'GET', "services/iscsi/target?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $targets = decode_json($response);
+ foreach my $t (@$targets) {
+ if ($t->{iscsi_target_name} eq "vm-$vmid") {
+ $target = $t->{id};
+ last;
+ }
+ }
+
+ return $target;
+}
+
+sub freenas_create_target {
+ my ($scfg, $vmid, $valid_code) = @_;
+ my $data;
+
+ freenas_get_version($scfg) unless $version;
+
+ if ($version < 110000) {
+ $data = {
+ iscsi_target_alias => "vm-$vmid",
+ iscsi_target_name => "vm-$vmid",
+ };
+ } elsif ($version < 110100) {
+ $data = {
+ iscsi_target_alias => "vm-$vmid",
+ iscsi_target_name => "vm-$vmid",
+ };
+ } else {
+ die "FreeNAS-$version: Unsupported!";
+ }
+ my $response = freenas_request($scfg, 'POST', "services/iscsi/target", encode_json($data), $valid_code);
+ if ($response =~ /^\d+$/) {
+ return freenas_get_target($scfg, $vmid) if $valid_code && $response == $valid_code;
+ die HTTP::Status::status_message($response);
+ }
+ my $target = decode_json($response);
+
+ die "Creating target for 'vm-$vmid' failed" unless $target->{id};
+
+ return $target->{id};
+}
+
+sub freenas_delete_target {
+ my ($scfg, $target) = @_;
+
+ my $response = freenas_request($scfg, 'DELETE', "services/iscsi/target/$target");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_target_name {
+ my ($scfg, $volname) = @_;
+ my $name = undef;
+
+ if ($volname =~ /^(vm|base)-(\d+)-/) {
+ $name = "vm-$2";
+ return "$target_prefix\:$name";
+ }
+
+ return undef;
+}
+
+sub freenas_create_target_group {
+ my ($scfg, $target, $valid_code) = @_;
+ my $data;
+
+ freenas_get_version($scfg) unless $version;
+
+ # Trying to create a target group which already exists will cause and internal
+ # server error so if creating an existing target group should be allowed (return
+ # existing target group number we must search prior to create
+ if ($valid_code && $valid_code == 409) {
+ my $tg = freenas_get_target_group($scfg, $target);
+ return $tg if $tg;
+ }
+
+ if ($version < 110000) {
+ $data = {
+ iscsi_target => $target,
+ iscsi_target_authgroup => $scfg->{auth_group} ? $scfg->{auth_group} : undef,
+ iscsi_target_portalgroup => $scfg->{portal_group},
+ iscsi_target_initiatorgroup => $scfg->{initiator_group},
+ iscsi_target_authtype => $scfg->{auth_type} ? $scfg->{auth_type} : 'None',
+ iscsi_target_initialdigest => "Auto",
+ };
+ } elsif ($version < 110100) {
+ $data = {
+ iscsi_target => $target,
+ iscsi_target_authgroup => $scfg->{auth_group} ? $scfg->{auth_group} : undef,
+ iscsi_target_portalgroup => $scfg->{portal_group},
+ iscsi_target_initiatorgroup => $scfg->{initiator_group},
+ iscsi_target_authtype => $scfg->{auth_type} ? $scfg->{auth_type} : 'None',
+ iscsi_target_initialdigest => "Auto",
+ };
+ } else {
+ die "FreeNAS-$version: Unsupported!";
+ }
+
+ my $response = freenas_request($scfg, 'POST', "services/iscsi/targetgroup", encode_json($data), $valid_code);
+ if ($response =~ /^\d+$/) {
+ if ($valid_code != 409) {
+ return freenas_get_target_group($scfg, $target) if $valid_code && $response == $valid_code;
+ }
+ die HTTP::Status::status_message($response);
+ }
+ my $tg = decode_json($response);
+
+ die "Creating target group for target '$target' failed" unless $tg->{id};
+
+ return $tg->{id};
+}
+
+sub freenas_delete_target_group {
+ my ($scfg, $tg) = @_;
+
+ my $response = freenas_request($scfg, 'DELETE', "services/iscsi/targetgroup/$tg");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_target_group {
+ my ($scfg, $target) = @_;
+ my $targetgroup = undef;
+
+ my $response = freenas_request($scfg, 'GET', "services/iscsi/targetgroup?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $targetgroups = decode_json($response);
+
+ foreach my $tgroup (@$targetgroups) {
+ if ($tgroup->{iscsi_target} == $target &&
+ $tgroup->{iscsi_target_portalgroup} == $scfg->{portal_group} &&
+ $tgroup->{iscsi_target_initiatorgroup} == $scfg->{initiator_group}) {
+ $targetgroup = $tgroup->{id};
+ last;
+ }
+ }
+
+ return $targetgroup;
+}
+
+sub freenas_create_extent {
+ my ($scfg, $zvol) = @_;
+ my $data;
+
+ freenas_get_version($scfg) unless $version;
+
+ if ($version < 110000) {
+ $data = {
+ iscsi_target_extent_type => 'Disk',
+ iscsi_target_extent_name => $zvol,
+ iscsi_target_extent_disk => "zvol/$scfg->{pool}/$zvol",
+ };
+ } elsif ($version < 110100) {
+ $data = {
+ iscsi_target_extent_type => 'Disk',
+ iscsi_target_extent_name => $zvol,
+ iscsi_target_extent_disk => "zvol/$scfg->{pool}/$zvol",
+ };
+ } else {
+ die "FreeNAS-$version: Unsupported!";
+ }
+
+ my $response = freenas_request($scfg, 'POST', "services/iscsi/extent", encode_json($data));
+ die HTTP::Status::status_message($response) if ($response =~ /^\d+$/);
+ my $extent = decode_json($response);
+
+ die "Creating LUN for volume '$zvol' failed" unless $extent->{id};
+
+ return $extent->{id};
+}
+
+sub freenas_delete_extent {
+ my ($scfg, $extent) = @_;
+
+ my $response = freenas_request($scfg, 'DELETE', "services/iscsi/extent/$extent");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_extent {
+ my ($scfg, $volname) = @_;
+ my $extent = undef;
+
+ my $response = freenas_request($scfg, 'GET', "services/iscsi/extent?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $extents = decode_json($response);
+ foreach my $ext (@$extents) {
+ if ($ext->{iscsi_target_extent_path} =~ /$scfg->{pool}\/$volname$/) {
+ $extent = $ext->{id};
+ last;
+ }
+ }
+
+ return $extent;
+}
+
+sub freenas_create_target_to_exent {
+ my ($scfg, $target, $extent, $lunid) = @_;
+ my $data;
+
+ freenas_get_version($scfg) unless $version;
+
+ if ($version < 110000) {
+ $data = {
+ iscsi_target => $target,
+ iscsi_extent => $extent,
+ iscsi_lunid => $lunid,
+ };
+ } elsif ($version < 110100) {
+ $data = {
+ iscsi_target => $target,
+ iscsi_extent => $extent,
+ iscsi_lunid => $lunid,
+ };
+ } else {
+ die "FreeNAS-$version: Unsupported!";
+ }
+
+ my $response = freenas_request($scfg, 'POST', "services/iscsi/targettoextent", encode_json($data));
+ die HTTP::Status::status_message($response) if ($response =~ /^\d+$/);
+ my $tg2extent = decode_json($response);
+
+ die "Creating view for LUN '$extent' failed" unless $tg2extent->{id};
+
+ return $tg2extent->{id};
+}
+
+sub freenas_delete_target_to_exent {
+ my ($scfg, $tg2exent) = @_;
+
+ my $response = freenas_request($scfg, 'DELETE', "services/iscsi/targettoextent/$tg2exent");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_target_to_exent {
+ my ($scfg, $extent, $target) = @_;
+ my $t2extent = undef;
+
+ my $response = freenas_request($scfg, 'GET', "services/iscsi/targettoextent?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $t2extents = decode_json($response);
+ foreach my $t2ext (@$t2extents) {
+ if ($t2ext->{iscsi_target} == $target && $t2ext->{iscsi_extent} == $extent) {
+ $t2extent = $t2ext->{id};
+ last;
+ }
+ }
+
+ return $t2extent;
+}
+
+sub freenas_find_free_diskname {
+ my ($storeid, $scfg, $vmid, $format) = @_;
+
+ my $name = undef;
+ my $volumes = freenas_list_zvol($scfg);
+
+ my $disk_ids = {};
+ my $dat = $volumes->{$scfg->{pool}};
+
+ foreach my $image (keys %$dat) {
+ my $volname = $dat->{$image}->{name};
+ if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
+ $disk_ids->{$2} = 1;
+ }
+ }
+
+ for (my $i = 1; $i < $max_luns + 1; $i++) {
+ if (!$disk_ids->{$i}) {
+ return "vm-$vmid-disk-$i";
+ }
+ }
+
+ die "Maximum number of LUNs($max_luns) for this VM $vmid in storage '$storeid' is reached.";
+}
+
+sub freenas_get_lun_number {
+ my ($scfg, $volname, $snap) = @_;
+ my $lunid = undef;
+
+ if ($volname =~ /^(vm|base)-\d+-disk-(\d+)$/ && ! defined $snap) {
+ $lunid = $2 - 1;
+ } elsif ($volname =~ /^vm-(\d+)-state/) {
+ # Find id for temporary LUN
+ if ($snap) {
+ # TODO
+ # Required to be able to exposed read-only LUNs for snapshot backup CT
+ } else {
+ my $target = freenas_get_target($scfg, $1);
+ my $id = $max_luns;
+ my $response = freenas_request($scfg, 'GET', "services/iscsi/targettoextent?limit=$limit");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $t2extents = decode_json($response);
+
+ foreach my $t2extent (@$t2extents) {
+ next unless $t2extent->{iscsi_target} == $target &&
+ $t2extent->{iscsi_lunid} + 1 > $max_luns &&
+ $t2extent->{iscsi_lunid} < $max_luns + $active_snaps;
+ my $eid = freenas_get_extent($scfg, $volname);
+ if ($eid) {
+ $response = freenas_request($scfg, 'GET', "services/iscsi/extent/$eid");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $extent = decode_json($response);
+ # Request to get lunid for an existing lun
+ last if $t2extent->{iscsi_extent} eq $eid;
+ }
+ $id++;
+ }
+ die "Max snapshots (4) is reached" unless ($id - $max_luns) < $active_snaps;
+ $lunid = $id;
+ }
+ }
+
+ return $lunid;
+}
+
+sub freenas_create_lun {
+ my ($scfg, $vmid, $zvol) = @_;
+ my ($target, $tg, $extent, $tg2exent) = (undef, undef, undef, undef);
+
+ eval {
+ $target = freenas_create_target($scfg, $vmid, 409);
+ die "create_lun-> Could not create target for VM '$vmid'" unless $target;
+ $tg = freenas_create_target_group($scfg, $target, 409);
+ die "create_lun-> Could not create target group for VM '$vmid'" unless $tg;
+ $extent = freenas_create_extent($scfg, $zvol);
+ die "create_lun-> Could not create extent for VM '$vmid'" unless $extent;
+ my $lunid = freenas_get_lun_number($scfg, $zvol);
+ die "create_lun-> $zvol: Bad name format for VM '$vmid'" unless defined $lunid;
+ $tg2exent = freenas_create_target_to_exent($scfg, $target, $extent, $lunid);
+ die "create_lun-> Could not create target to extend for VM '$vmid'" unless defined $tg2exent;
+ };
+ if ($@) {
+ my $err = $@;
+ if ($tg2exent) {
+ freenas_delete_target_to_exent($scfg, $tg2exent);
+ }
+ if ($extent) {
+ freenas_delete_extent($scfg, $extent);
+ }
+ if ($target && freenas_no_more_extents($scfg, $target)) {
+ if ($tg) {
+ freenas_delete_target_group($scfg, $tg);
+ }
+ freenas_delete_target($scfg, $target);
+ }
+ die $err;
+ }
+}
+
+sub freenas_create_zvol {
+ my ($scfg, $volname, $size) = @_;
+
+ my $data = {
+ name => $volname,
+ volsize => $size,
+ };
+ my $response = freenas_request($scfg, 'POST', "storage/volume/$scfg->{pool}/zvols", encode_json($data));
+ die HTTP::Status::status_message($response) if ($response =~ /^\d+$/);
+ my $zvol = decode_json($response);
+
+ die "$volname: Failed creating volume" unless $zvol && $zvol->{name};
+
+ return $zvol->{name};
+}
+
+sub freenas_delete_zvol {
+ my ($scfg, $volname) = @_;
+
+ my $response = freenas_request($scfg, 'DELETE', "storage/volume/$scfg->{pool}/zvols/$volname");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub os_request {
+ my ($cmd, $noerr, $timeout) = @_;
+
+ $timeout = PVE::RPCEnvironment::is_worker() ? 60*60 : 5 if !$timeout;
+ $noerr = 0 if !$noerr;
+
+ my $text = '';
+
+ my $output = sub {
+ my $line = shift;
+ $text .= "$line\n";
+ };
+
+ my $exit_code = run_command($cmd, noerr => $noerr, errfunc => $output, outfunc => $output, timeout => $timeout);
+
+ return wantarray ? ($exit_code, $text) : $exit_code;
+}
+
+sub bail_out {
+ my ($class, $storeid, $scfg, $volname, $err) = @_;
+
+ $class->free_image($storeid, $scfg, $volname);
+ die $err;
+}
+
+sub disk_by_path {
+ my ($scfg, $volname) = @_;
+
+ my $target = freenas_get_target_name($scfg, $volname);
+ my $lun = freenas_get_lun_number($scfg, $volname);
+ my $path = "/dev/disk/by-path/ip-$scfg->{portal}\:3260-iscsi-$target-lun-$lun";
+
+ return $path;
+}
+
+sub build_lun_list {
+ my ($scfg, $sid, $lun) = @_;
+
+ my $luns = {};
+ my $text = '';
+ my $exit = 0;
+
+ eval {
+ ($exit, $text) = os_request("iscsiadm -m session -r $sid -P3", 1, 60);
+ };
+ if ($@) {
+ # An exist code of 22 means no active session otherwise an error
+ if ($exit != 22) {
+ die "$@";
+ }
+ }
+ if ($text =~ /.*Host Number:\s*(\d+)\s+State:\s+running(.*)/s) {
+ my $host = $1;
+ my $found = 0;
+ for (split /^/, $2) {
+ if ($_ =~ /Channel\s+(\d+)\s+Id\s+(\d+)\s+Lun:\s+(\d+)/) {
+ if (defined $lun && $lun == $3) {
+ $luns = {};
+ $found = 1;
+ }
+ $luns->{$3} = "$host:".int($1).":$2:$3";
+ last if $found;
+ }
+ }
+ }
+
+ return $luns;
+}
+
+sub get_sid {
+ my ($scfg, $volname) = @_;
+ my $sid = -1;
+ my $text = '';
+ my $exit = 0;
+
+ my $target = freenas_get_target_name($scfg, $volname);
+
+ eval {
+ ($exit, $text) = os_request("iscsiadm -m node -T $target -p $scfg->{portal} -s", 1, 60);
+ };
+ if ($@) {
+ # An exist code of 21 or 22 means no active session otherwise an error
+ if ($exit != 21 || $exit != 22) {
+ die "$@";
+ }
+ }
+ if ($text =~ /.*\[sid\:\s*(\d+),\s*.*/) {
+ $sid = $1;
+ }
+
+ return $sid;
+}
+
+sub create_session {
+ my ($scfg, $volname) = @_;
+ my $sid = -1;
+ my $exit = undef;
+
+ my $target = freenas_get_target_name($scfg, $volname);
+
+ eval {
+ $exit = os_request("iscsiadm -m node -T $target -p $scfg->{portal} --login", 1, 60);
+ if ($exit == 21) {
+ eval {
+ os_request("iscsiadm -m discovery -t sendtargets -p $scfg->{portal}", 0, 60);
+ os_request("iscsiadm -m node -T $target -p $scfg->{portal} --login", 0, 60);
+ };
+ }
+ };
+ if ($@) {
+ if ($exit == 21) {
+ eval {
+ os_request("iscsiadm -m discovery -t sendtargets -p $scfg->{portal}", 0, 60);
+ os_request("iscsiadm -m node -T $target -p $scfg->{portal} --login", 0, 60);
+ };
+ } else {
+ die $@;
+ }
+ }
+ eval {
+ $sid = get_sid($scfg, $volname);
+ };
+ die "$@" if $@;
+ die "Could not create session" if $sid < 0;
+
+ return $sid;
+}
+
+sub delete_session {
+ my ($scfg, $sid) = @_;
+
+ eval {
+ os_request("iscsiadm -m session -r $sid --logout", 0, 60);
+ };
+}
+
+sub remove_local_lun {
+ my ($id) = @_;
+
+ os_request("echo 1 > /sys/bus/scsi/devices/$id/delete", 0, 60);
+}
+
+sub deactivate_luns {
+ # $luns contains a hash of luns to keep
+ my ($scfg, $volname, $luns) = @_;
+
+ $luns = {} if !$luns;
+ my $sid;
+ my $list = {};
+
+ eval {
+ $sid = get_sid($scfg, $volname);
+ };
+ die "$@" if $@;
+
+ eval {
+ $list = build_lun_list($scfg, $sid);
+
+ foreach my $key (keys %$list) {
+ next if exists($luns->{$key});
+ remove_local_lun($list->{$key});
+ }
+ };
+ die "$@" if $@;
+}
+
+sub get_active_luns {
+ my ($class, $storeid, $scfg, $volname) = @_;
+
+ my $sid = 0;
+ my $luns = {};
+
+ eval {
+ $sid = get_sid($scfg, $volname);
+ };
+ die "$@" if $@;
+ if ($sid < 0) {
+ # We have no active sessions so make one
+ eval {
+ $sid = create_session($scfg, $volname);
+ };
+ die "$@" if $@;
+ # Since no session existed prior to this call deactivate all LUN's found
+ deactivate_luns($scfg, $volname);
+ } else {
+ eval {
+ $luns = build_lun_list($scfg, $sid);
+ };
+ die "$@" if $@;
+ }
+
+ return $luns;
+}
+
+sub rescan_session {
+ my ($class, $storeid, $scfg, $volname, $exclude_lun) = @_;
+
+ eval {
+ my $active_luns = get_active_luns($class, $storeid, $scfg, $volname);
+ delete $active_luns->{$exclude_lun} if defined $exclude_lun;
+ my $sid = get_sid($scfg, $volname);
+ die "Missing session" if $sid < 0;
+ os_request("iscsiadm -m session -r $sid -R", 0, 60);
+ deactivate_luns($scfg, $volname, $active_luns);
+ delete_session($scfg, $sid) if !%$active_luns;
+ };
+ die "$@" if $@;
+}
+
+sub freenas_get_latest_snapshot {
+ my ($class, $scfg, $volname) = @_;
+
+ my $vname = ($class->parse_volname($volname))[1];
+
+ # abort rollback if snapshot is not the latest
+ my $response = freenas_request($scfg, 'GET', "storage/snapshot");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $snapshots = decode_json($response);
+
+ my $recentsnap;
+ foreach my $snapshot (@$snapshots) {
+ next unless $snapshot->{filesystem} =~ /$scfg->{pool}\/$vname/ && $snapshot->{mostrecent};
+ $recentsnap = $snapshot->{name};
+ last;
+ }
+
+ return $recentsnap;
+}
+
+# Configuration
+
+sub type {
+ return 'freenas';
+}
+
+sub plugindata {
+ return {
+ content => [ {images => 1, rootdir => 1}, {images => 1 , rootdir => 1} ],
+ format => [ { raw => 1 } , 'raw' ],
+ };
+}
+
+sub properties {
+ return {
+ password => {
+ description => "password",
+ type => "string",
+ },
+ portal_group => {
+ description => "Portal Group ID",
+ type => "integer",
+ },
+ initiator_group => {
+ description => "Initiator Group ID",
+ type => "integer",
+ },
+ };
+}
+
+sub options {
+ return {
+ portal => { fixed => 1 },
+ pool => { fixed => 1 },
+ portal_group => { fixed => 1 },
+ initiator_group => { fixed => 1 },
+ blocksize => { optional => 1 },
+ username => { optional => 1 },
+ password => { optional => 1 },
+# sparse => { optional => 1 }, not available in 9.2.x. Appear in 11.x
+# in 9.2.x all zvols are created sparse!
+ nodes => { optional => 1 },
+ disable => { optional => 1 },
+ content => { optional => 1 },
+ };
+}
+
+# Storage implementation
+
+sub volume_size_info {
+ my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+ my (undef, $vname) = $class->parse_volname($volname);
+
+ my $response = freenas_request($scfg, 'GET', "storage/volume/$scfg->{pool}/zvols/$vname");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $zvol = decode_json($response);
+
+ return $zvol->{volsize} if $zvol && $zvol->{volsize};
+
+ die "Could not get zfs volume size\n";
+}
+
+sub parse_volname {
+ my ($class, $volname) = @_;
+
+ if ($volname =~ m/^(((base)-(\d+)-\S+)\/)?((base|vm)-(\d+)-\S+)$/) {
+ my $format = 'raw';
+ my $isBase = ($6 eq 'base');
+ return ('images', $5, $7, $2, $4, $isBase, $format);
+ }
+
+ die "unable to parse freenas volume name '$volname'\n";
+}
+
+sub status {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ my $total = 0;
+ my $free = 0;
+ my $used = 0;
+ my $active = 0;
+
+ eval {
+ my $response = freenas_request($scfg, 'GET', "storage/volume/$scfg->{pool}");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $vol = decode_json($response);
+ my $children = $vol->{children};
+ if (@$children) {
+ $used = $children->[0]{used};
+ $total = $children->[0]{avail};
+ } else {
+ $used = $vol->{used};
+ $total = $vol->{avail};
+ }
+ $free = $total - $used;
+ $active = 1;
+ };
+ warn $@ if $@;
+
+ return ($total, $free, $used, $active);
+}
+
+sub list_images {
+ my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+
+ $cache->{freenas} = freenas_list_zvol($scfg) if !$cache->{freenas};
+ my $zfspool = $scfg->{pool};
+ my $res = [];
+
+ if (my $dat = $cache->{freenas}->{$zfspool}) {
+
+ foreach my $image (keys %$dat) {
+
+ my $info = $dat->{$image};
+ my $volname = $info->{name};
+ my $parent = $info->{parent};
+ my $owner = $info->{vmid};
+
+ if ($parent) {
+ $info->{volid} = "$storeid:$parent/$volname";
+ } else {
+ $info->{volid} = "$storeid:$volname";
+ }
+
+ if ($vollist) {
+ my $found = grep { $_ eq $info->{volid} } @$vollist;
+ next if !$found;
+ } else {
+ next if defined ($vmid) && ($owner ne $vmid);
+ }
+ push @$res, $info;
+ }
+ }
+
+ return $res;
+}
+
+sub path {
+ my ($class, $scfg, $volname, $storeid, $snapname) = @_;
+
+ die "direct access to snapshots not implemented"
+ if defined($snapname);
+
+ my ($vtype, $vname, $vmid) = $class->parse_volname($volname);
+
+ my $luns = get_active_luns($class, $storeid, $scfg, $vname);
+ my $path = disk_by_path($scfg, $vname);
+
+ return ($path, $vmid, $vtype);
+}
+
+sub create_base {
+ my ($class, $storeid, $scfg, $volname) = @_;
+ my ($lun, $target);
+ my $snap = '__base__';
+
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+ $class->parse_volname($volname);
+
+ die "create_base not possible with base image\n" if $isBase;
+
+ my $newname = $name;
+ $newname =~ s/^vm-/base-/;
+
+ $target = freenas_get_target($scfg, $vmid);
+ die "create_base-> missing target" unless $target;
+ my $extent = freenas_get_extent($scfg, $name);
+ die "create_base-> missing extent" unless $extent;
+ my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+ die "create_base-> missing target to extent" unless $tg2exent;
+ $lun = freenas_get_lun_number($scfg, $name);
+ die "create_base-> missing LUN" unless defined $lun;
+ freenas_delete_target_to_exent($scfg, $tg2exent);
+ freenas_delete_extent($scfg, $extent);
+ my $sid = get_sid($scfg, $name);
+ if ($sid >= 0) {
+ my $lid = build_lun_list($scfg, $sid, $lun);
+ if ($lid && $lid->{$lun}) {
+ remove_local_lun($lid->{$lun});
+ }
+ }
+
+ eval {
+ # FreeNAS API does not support renaming a zvol so create a snapshot
+ # and make a clone of the snapshot instead
+ $class->volume_snapshot($scfg, $storeid, $name, $snap);
+
+ my $data = {
+ name => "$scfg->{pool}/$newname"
+ };
+ my $response = freenas_request($scfg, 'POST', "storage/snapshot/$scfg->{pool}/$name\@$snap/clone/", encode_json($data));
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+
+ freenas_create_lun($scfg, $vmid, $newname);
+ };
+ if ($@) {
+ $extent = freenas_create_extent($scfg, $name);
+ die "create_base-> Could not create extent for VM '$vmid'" unless $extent;
+ $tg2exent = freenas_create_target_to_exent($scfg, $target, $extent, $lun);
+ die "create_base-> Could not create target to extend for VM '$vmid'" unless defined $tg2exent;
+ }
+
+ return $newname;
+}
+
+sub clone_image {
+ my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
+
+ $snap ||= "__base__$vmid";
+
+ my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
+ $class->parse_volname($volname);
+
+ die "clone_image only works on base images" if !$isBase;
+
+ my $run = PVE::QemuServer::check_running($basevmid);
+ if (!$run) {
+ $run = PVE::LXC::check_running($basevmid);
+ }
+
+ my $name = freenas_find_free_diskname($storeid, $scfg, $vmid, $format);
+
+ $class->volume_snapshot($scfg, $storeid, $basename, $snap);
+
+ my $data = {
+ name => "$scfg->{pool}/$name"
+ };
+ my $response = freenas_request($scfg, 'POST', "storage/snapshot/$scfg->{pool}/$basename\@$snap/clone/", encode_json($data));
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+
+ $name = "$basename/$name";
+ # get ZFS dataset name from PVE volname
+ my (undef, $clonedname) = $class->parse_volname($name);
+
+ freenas_create_lun($scfg, $vmid, $clonedname);
+
+ my $res = $class->deactivate_volume($storeid, $scfg, $basename) unless $run;
+ warn "Could not deactivate volume '$basename'" unless $res;
+
+ return $name;
+}
+
+sub alloc_image {
+ my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+ die "unsupported format '$fmt'" if $fmt ne 'raw';
+
+ die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
+ if $name && $name !~ m/^vm-$vmid-/;
+
+ my $volname = $name;
+
+ $volname = freenas_find_free_diskname($storeid, $scfg, $vmid, $fmt) if !$volname;
+
+ # Size is in KB but Freenas wants in bytes
+ $size *= 1024;
+ my $zvol = freenas_create_zvol($scfg, $volname, $size);
+
+ eval {
+ freenas_create_lun($scfg, $vmid, $zvol) if $zvol;
+ };
+ if ($@) {
+ my $err = $@;
+ eval {
+ freenas_delete_zvol($scfg, $volname);
+ };
+ $err .= "\n$@" if $@;
+ die $err;
+ }
+
+ return $volname;
+}
+
+sub free_image {
+ my ($class, $storeid, $scfg, $volname, $isBase) = @_;
+
+ my ($vtype, $name, $vmid, $basename) = $class->parse_volname($volname);
+
+ eval {
+ my $target = freenas_get_target($scfg, $vmid);
+ die "free_image-> missing target" unless $target;
+ my $extent = freenas_get_extent($scfg, $name);
+ die "free_image-> missing extent" unless $extent;
+ my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+ die "free_image-> missing target to extent" unless $tg2exent;
+ my $target_group = freenas_get_target_group($scfg, $target);
+ die "free_image-> missing target group" unless $target_group;
+ my $lun = freenas_get_lun_number($scfg, $name);
+ die "free_image-> missing LUN" unless defined $lun;
+
+ my $res = $class->deactivate_volume($storeid, $scfg, $volname);
+ warn "Could not deactivate volume '$volname'" unless $res;
+ freenas_delete_target_to_exent($scfg, $tg2exent);
+ freenas_delete_extent($scfg, $extent);
+ if ($target && freenas_no_more_extents($scfg, $target)) {
+ if ($target_group) {
+ freenas_delete_target_group($scfg, $target_group);
+ }
+ freenas_delete_target($scfg, $target);
+ }
+ freenas_delete_zvol($scfg, $name);
+ $class->volume_snapshot_delete($scfg, $storeid, $basename, "__base__$vmid") if $basename;
+ if ($isBase) {
+ $basename = $name;
+ $basename =~ s/^base-/vm-/;
+ $class->volume_snapshot_delete($scfg, $storeid, $basename, '__base__') if $basename;
+ freenas_delete_zvol($scfg, $basename);
+ }
+ };
+ if ($@) {
+ my $err = $@;
+ freenas_create_lun($scfg, $vmid, $name) unless $isBase;
+ die $err;
+ }
+
+ return undef;
+}
+
+sub volume_resize {
+ my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
+
+ die 'mode failure - unable to resize disk(s) on a running system due to FreeNAS bug.<br />
+ See bug report: <a href="https://bugs.freenas.org/issues/24432" target="_blank">#24432</a><br />' if $running;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ my $data = {
+ volsize => $size,
+ };
+ my $response = freenas_request($scfg, 'PUT', "storage/volume/$scfg->{pool}/zvols/$name", encode_json($data));
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+ my $vol = decode_json($response);
+
+ my $sid = get_sid($scfg, $name);
+ if ($sid >= 0) {
+ eval {
+#### Required because of a bug in FreeNAS: https://bugs.freenas.org/issues/24432
+ my $targetname = freenas_get_target_name($scfg, $name);
+ die "volume_resize-> Missing target name" unless $targetname;
+ my $target = freenas_get_target($scfg, $vmid);
+ die "volume_resize-> Missing target" unless $target;
+ my $extent = freenas_get_extent($scfg, $name);
+ die "volume_resize-> Missing extent" unless $extent;
+ my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+ die "volume_resize-> Missing target to extent" unless $tg2exent;
+ my $lunid = freenas_get_lun_number($scfg, $name);
+ die "volume_resize-> Missing LUN" unless defined $lunid;
+ freenas_delete_target_to_exent($scfg, $tg2exent);
+ freenas_create_target_to_exent($scfg, $target, $extent, $lunid);
+#### Required because of a bug in FreeNAS: https://bugs.freenas.org/issues/24432
+ rescan_session($class, $storeid, $scfg, $name, $lunid);
+ };
+ die "$name: Resize with $size failed. ($@)\n" if $@;
+ }
+
+ return int($vol->{volsize}/1024);
+}
+
+sub volume_snapshot {
+ my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+ my $vname = ($class->parse_volname($volname))[1];
+
+ my $data = {
+ dataset => "$scfg->{pool}/$vname",
+ name => $snap,
+ };
+ my $response = freenas_request($scfg, 'POST', "storage/snapshot", encode_json($data));
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub volume_snapshot_delete {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ my $vname = ($class->parse_volname($volname))[1];
+
+ my $response = freenas_request($scfg, 'DELETE', "storage/snapshot/$scfg->{pool}/$vname\@$snap");
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub volume_snapshot_rollback {
+ my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ my $target = freenas_get_target($scfg, $vmid);
+ die "volume_resize-> Missing target" unless $target;
+ my $extent = freenas_get_extent($scfg, $name);
+ die "volume_resize-> Missing extent" unless $extent;
+ my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+ die "volume_resize-> Missing target to extent" unless $tg2exent;
+ my $lunid = freenas_get_lun_number($scfg, $name);
+ die "volume_resize-> Missing LUN" unless defined $lunid;
+ freenas_delete_target_to_exent($scfg, $tg2exent);
+ freenas_delete_extent($scfg, $extent);
+
+ my $data = {
+ force => bless( do{\(my $o = 0)}, 'JSON::XS::Boolean' ),
+ };
+ my $response = freenas_request($scfg, 'POST', "storage/snapshot/$scfg->{pool}/$name\@$snap/rollback/", encode_json($data));
+ die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+
+ $extent = freenas_create_extent($scfg, $name);
+ freenas_create_target_to_exent($scfg, $target, $extent, $lunid);
+ rescan_session($class, $storeid, $scfg, $name, $lunid);
+}
+
+sub volume_rollback_is_possible {
+ my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+ my (undef, $name) = $class->parse_volname($volname);
+
+ my $recentsnap = $class->freenas_get_latest_snapshot($scfg, $name);
+ if ($snap ne $recentsnap) {
+ die "can't rollback, more recent snapshots exist";
+ }
+
+ return 1;
+}
+
+sub volume_snapshot_list {
+ my ($class, $scfg, $storeid, $volname, $prefix) = @_;
+ # return an empty array if dataset does not exist.
+ die "Volume_snapshot_list is not implemented for FreeNAS.\n";
+}
+
+sub volume_has_feature {
+ my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+ my $features = {
+ snapshot => { current => 1, snap => 1},
+ clone => { base => 1},
+ template => { current => 1},
+ copy => { base => 1, current => 1},
+ };
+
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+ $class->parse_volname($volname);
+
+ my $key = undef;
+
+ if ($snapname) {
+ $key = 'snap';
+ } else {
+ $key = $isBase ? 'base' : 'current';
+ }
+
+ return 1 if $features->{$feature}->{$key};
+
+ return undef;
+}
+
+sub activate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ return 1;
+}
+
+sub deactivate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ return 1;
+}
+
+# Procedure for activating a LUN:
+#
+# if session does not exist
+# login to target
+# deactivate all luns in session
+# get list of active luns
+# get lun number to activate
+# make list of our luns (active + new lun)
+# rescan session
+# deactivate all luns except our luns
+sub activate_volume {
+ my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+
+ # TODO: FreeNAS supports exposing a ro LUN from a snapshot
+ # ERROR: Backup of VM xyz failed - unable to activate snapshot from remote zfs storage
+ # at /usr/share/perl5/PVE/Storage/ZFSPlugin.pm line 418.
+ # $vname\@$snapname
+ die "mode failure - unable to activate snapshot from remote zfs storage" if $snapname;
+
+ my (undef, $name) = $class->parse_volname($volname);
+
+ my $active_luns = get_active_luns($class, $storeid, $scfg, $name);
+
+ my $lun = freenas_get_lun_number($scfg, $name);
+ $active_luns->{$lun} = "0:0:0:$lun";
+
+ eval {
+ my $sid = get_sid($scfg, $name);
+ die "activate_volume-> Missing session" if $sid < 0;
+ # Add new LUN's to session
+ os_request("iscsiadm -m session -r $sid -R", 0, 60);
+ sleep 1;
+ # Remove all LUN's from session which is not currently active
+ deactivate_luns($scfg, $name, $active_luns);
+ };
+ die "$@" if $@;
+
+ return 1;
+}
+
+# Procedure for deactivating a LUN:
+#
+# if session exists
+# get lun number to deactivate
+# deactivate lun
+sub deactivate_volume {
+ my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+
+ # TODO: FreeNAS supports exposing a ro LUN from a snapshot
+ # ERROR: Backup of VM xyz failed - unable to activate snapshot from remote zfs storage
+ # at /usr/share/perl5/PVE/Storage/ZFSPlugin.pm line 418.
+ # $vname\@$snapname
+ die "mode failure - unable to deactivate snapshot from remote zfs storage" if $snapname;
+
+ my (undef, $name) = $class->parse_volname($volname);
+
+ my $active_luns = get_active_luns($class, $storeid, $scfg, $name);
+
+ my $lun = freenas_get_lun_number($scfg, $name);
+ delete $active_luns->{$lun};
+
+ eval {
+ my $sid = get_sid($scfg, $name);
+ die "deactivate_volume-> Missing session" if $sid < 0;
+ deactivate_luns($scfg, $name, $active_luns);
+ delete_session($scfg, $sid) if !%$active_luns;
+ };
+ die $@ if $@;
+
+ return 1;
+}
+
+1;
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index b924f21..23a4aa8 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm FreeNASPlugin.pm
.PHONY: install
install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 4df6608..d827bbb 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -325,7 +325,7 @@ sub parse_config {
$d->{content} = $def->{content}->[1] if !$d->{content};
}
- if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'glusterfs' || $type eq 'zfs' || $type eq 'drbd') {
+ if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'glusterfs' || $type eq 'zfs' || $type eq 'drbd' || $type eq 'freenas') {
$d->{shared} = 1;
}
}
--
2.11.0
More information about the pve-devel
mailing list