[pve-devel] [PATCH storage 1/1] Cephfs storage plugin

Alexandre DERUMIER aderumier at odiso.com
Fri Apr 20 15:42:22 CEST 2018


Hi,

+
+sub plugindata {
+    return {
+        content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1},
+                     { images => 1 }],
+        format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ],
+    };
+} 


I think we should forbid images, as I'm pretty sure that users will try it.

----- Mail original -----
De: "Alwin Antreich" <a.antreich at proxmox.com>
À: "pve-devel" <pve-devel at pve.proxmox.com>
Envoyé: Vendredi 20 Avril 2018 15:32:36
Objet: [pve-devel] [PATCH storage 1/1] Cephfs storage plugin

- ability to mount through kernel and fuse client 
- allow mount options 
- get MONs from ceph config if not in storage.cfg 
- allow the use of ceph config with fuse client 

Signed-off-by: Alwin Antreich <a.antreich at proxmox.com> 
--- 
PVE/API2/Storage/Config.pm | 2 +- 
PVE/API2/Storage/Status.pm | 2 +- 
PVE/Storage.pm | 2 + 
PVE/Storage/CephFSPlugin.pm | 262 ++++++++++++++++++++++++++++++++++++++++++++ 
PVE/Storage/Makefile | 2 +- 
PVE/Storage/Plugin.pm | 1 + 
debian/control | 2 + 
7 files changed, 270 insertions(+), 3 deletions(-) 
create mode 100644 PVE/Storage/CephFSPlugin.pm 

diff --git a/PVE/API2/Storage/Config.pm b/PVE/API2/Storage/Config.pm 
index 3b38304..368a5c9 100755 
--- a/PVE/API2/Storage/Config.pm 
+++ b/PVE/API2/Storage/Config.pm 
@@ -171,7 +171,7 @@ __PACKAGE__->register_method ({ 
PVE::Storage::activate_storage($cfg, $baseid); 

PVE::Storage::LVMPlugin::lvm_create_volume_group($path, $opts->{vgname}, $opts->{shared}); 
- } elsif ($type eq 'rbd' && !defined($opts->{monhost})) { 
+ } elsif (($type eq 'rbd' || $type eq 'cephfs') && !defined($opts->{monhost})) { 
my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring'; 
my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring"; 

diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm 
index ab07146..2d8d143 100644 
--- a/PVE/API2/Storage/Status.pm 
+++ b/PVE/API2/Storage/Status.pm 
@@ -335,7 +335,7 @@ __PACKAGE__->register_method ({ 
my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node); 

die "cant upload to storage type '$scfg->{type}'\n" 
- if !($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs'); 
+ if !($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' || $scfg->{type} eq 'cephfs'); 

my $content = $param->{content}; 

diff --git a/PVE/Storage.pm b/PVE/Storage.pm 
index d733380..f9732fe 100755 
--- a/PVE/Storage.pm 
+++ b/PVE/Storage.pm 
@@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin; 
use PVE::Storage::CIFSPlugin; 
use PVE::Storage::ISCSIPlugin; 
use PVE::Storage::RBDPlugin; 
+use PVE::Storage::CephFSPlugin; 
use PVE::Storage::SheepdogPlugin; 
use PVE::Storage::ISCSIDirectPlugin; 
use PVE::Storage::GlusterfsPlugin; 
@@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register(); 
PVE::Storage::CIFSPlugin->register(); 
PVE::Storage::ISCSIPlugin->register(); 
PVE::Storage::RBDPlugin->register(); 
+PVE::Storage::CephFSPlugin->register(); 
PVE::Storage::SheepdogPlugin->register(); 
PVE::Storage::ISCSIDirectPlugin->register(); 
PVE::Storage::GlusterfsPlugin->register(); 
diff --git a/PVE/Storage/CephFSPlugin.pm b/PVE/Storage/CephFSPlugin.pm 
new file mode 100644 
index 0000000..614a88f 
--- /dev/null 
+++ b/PVE/Storage/CephFSPlugin.pm 
@@ -0,0 +1,262 @@ 
+package PVE::Storage::CephFSPlugin; 
+ 
+use strict; 
+use warnings; 
+use IO::File; 
+use Net::IP; 
+use File::Path; 
+use PVE::Tools qw(run_command); 
+use PVE::ProcFSTools; 
+use PVE::Storage::Plugin; 
+use PVE::JSONSchema qw(get_standard_option); 
+ 
+use base qw(PVE::Storage::Plugin); 
+ 
+my $hostlist = sub { 
+ my ($list_text, $separator) = @_; 
+ 
+ my @monhostlist = PVE::Tools::split_list($list_text); 
+ return join($separator, map { 
+ my ($host, $port) = PVE::Tools::parse_host_and_port($_); 
+ $port = defined($port) ? ":$port" : ''; 
+ $host = "[$host]" if Net::IP::ip_is_ipv6($host); 
+ "${host}${port}" 
+ } @monhostlist); 
+}; 
+ 
+my $parse_ceph_config = sub { 
+ my ($filename) = @_; 
+ 
+ my $cfg = {}; 
+ 
+ return $cfg if ! -f $filename; 
+ 
+ my $fh = IO::File->new($filename, "r") || 
+ die "unable to open '$filename' - $!\n"; 
+ 
+ my $section; 
+ 
+ while (defined(my $line = <$fh>)) { 
+ $line =~ s/[;#].*$//; 
+ $line =~ s/^\s+//; 
+ $line =~ s/\s+$//; 
+ next if !$line; 
+ 
+ $section = $1 if $line =~ m/^\[(\S+)\]$/; 
+ if (!$section) { 
+ warn "no section - skip: $line\n"; 
+ next; 
+ } 
+ 
+ if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) { 
+ $cfg->{$section}->{$1} = $2; 
+ } 
+ 
+ } 
+ 
+ return $cfg; 
+}; 
+ 
+my $get_monaddr_list = sub { 
+ my ($scfg, $configfile) = @_; 
+ 
+ my $server; 
+ my $no_mon = !defined($scfg->{monhost}); 
+ 
+ if (($no_mon) && defined($configfile)) { 
+ my $config = $parse_ceph_config->($configfile); 
+ $server = join(',', sort { $a cmp $b } 
+ map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config}); 
+ }else { 
+ $server = $hostlist->($scfg->{monhost}, ','); 
+ } 
+ 
+ return $server; 
+}; 
+ 
+my $get_configfile = sub { 
+ my ($storeid) = @_; 
+ 
+ my $configfile; 
+ my $pve_cephconfig = '/etc/pve/ceph.conf'; 
+ my $storeid_cephconfig = "/etc/pve/priv/ceph/${storeid}.conf"; 
+ 
+ if (-e $pve_cephconfig) { 
+ if (-e $storeid_cephconfig) { 
+ warn "ignoring custom ceph config for storage '$storeid', 'monhost' is not set (assuming pveceph managed cluster)!\n"; 
+ } 
+ $configfile = $pve_cephconfig; 
+ } elsif (-e $storeid_cephconfig) { 
+ $configfile = $storeid_cephconfig; 
+ } else { 
+ die "Missing ceph config for ${storeid} storage\n"; 
+ } 
+ 
+ return $configfile; 
+}; 
+ 
+sub cephfs_is_mounted { 
+ my ($scfg, $storeid, $mountdata) = @_; 
+ 
+ my $no_mon = !defined($scfg->{monhost}); 
+ 
+ my $configfile = $get_configfile->($storeid) if ($no_mon); 
+ my $server = $get_monaddr_list->($scfg, $configfile); 
+ 
+ my $subdir = $scfg->{subdir} ? $scfg->{subdir} : '/'; 
+ my $mountpoint = $scfg->{path}; 
+ my $source = "$server:$subdir"; 
+ 
+ $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata; 
+ return $mountpoint if grep { 
+ $_->[2] =~ /^ceph|fuse\.ceph-fuse/ && 
+ $_->[0] =~ m#^\Q$source\E|ceph-fuse$# && 
+ $_->[1] eq $mountpoint 
+ } @$mountdata; 
+ 
+ warn "A filesystem is still mounted on $mountpoint\n" 
+ if grep { $_->[1] eq $mountpoint } @$mountdata; 
+ 
+ return undef; 
+} 
+ 
+sub cephfs_mount { 
+ my ($scfg, $storeid) = @_; 
+ 
+ my $cmd; 
+ 
+ my $no_mon = !defined($scfg->{monhost}); 
+ 
+ my $keyfile = "/etc/pve/priv/ceph/${storeid}.keyring"; 
+ my $username = $scfg->{username} ? $scfg->{username} : 'admin'; 
+ my $mountpoint = $scfg->{path}; 
+ my $subdir = $scfg->{subdir} ? $scfg->{subdir} : '/'; 
+ 
+ my $configfile = $get_configfile->($storeid) if ($no_mon); 
+ my $server = $get_monaddr_list->($scfg, $configfile); 
+ 
+ # fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg 
+ # kernel -> better performance, less frequent updates 
+ if ($scfg->{fuse}) { 
+ # FIXME: ceph-fuse client complains about missing ceph.conf or keyring if 
+ # not provided on its default locations but still connects. Fix upstream?? 
+ $cmd = ['/usr/bin/ceph-fuse', '-n', "client.$username", '-m', $server]; 
+ push @$cmd, '--keyfile', $keyfile if (-e $keyfile); 
+ push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|); 
+ push @$cmd, $mountpoint; 
+ push @$cmd, '--conf', $configfile if defined($configfile); 
+ }else { 
+ my $source = "$server:$subdir"; 
+ $cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', "name=$username"]; 
+ push @$cmd, '-o', "secretfile=$keyfile" if (-e $keyfile); 
+ } 
+ 
+ if ($scfg->{options}) { 
+ push @$cmd, '-o', $scfg->{options}; 
+ } 
+ 
+ run_command($cmd, errmsg => "mount error"); 
+} 
+ 
+# Configuration 
+ 
+sub type { 
+ return 'cephfs'; 
+} 
+ 
+sub plugindata { 
+ return { 
+ content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1}, 
+ { images => 1 }], 
+ format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ], 
+ }; 
+} 
+ 
+sub properties { 
+ return { 
+ fuse => { 
+ description => "Mount cephfs through fuse.", 
+ type => 'boolean', 
+ }, 
+ subdir => { 
+ description => "Subdir to mount.", 
+ type => 'string', format => 'pve-storage-path', 
+ }, 
+ }; 
+} 
+ 
+sub options { 
+ return { 
+ path => { fixed => 1 }, 
+ monhost => { optional => 1}, 
+ nodes => { optional => 1 }, 
+ subdir => { optional => 1 }, 
+ disable => { optional => 1 }, 
+ options => { optional => 1 }, 
+ username => { optional => 1 }, 
+ content => { optional => 1 }, 
+ format => { optional => 1 }, 
+ mkdir => { optional => 1 }, 
+ fuse => { optional => 1 }, 
+ bwlimit => { optional => 1 }, 
+ }; 
+} 
+ 
+sub check_config { 
+ my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_; 
+ 
+ $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path}; 
+ 
+ return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck); 
+} 
+ 
+# Storage implementation 
+ 
+sub status { 
+ my ($class, $storeid, $scfg, $cache) = @_; 
+ 
+ $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() 
+ if !$cache->{mountdata}; 
+ 
+ return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata}); 
+ 
+ return $class->SUPER::status($storeid, $scfg, $cache); 
+} 
+ 
+sub activate_storage { 
+ my ($class, $storeid, $scfg, $cache) = @_; 
+ 
+ $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() 
+ if !$cache->{mountdata}; 
+ 
+ my $path = $scfg->{path}; 
+ 
+ if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) { 
+ 
+ # NOTE: only call mkpath when not mounted (avoid hang 
+ # when cephfs is offline 
+ 
+ mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir}); 
+ 
+ die "unable to activate storage '$storeid' - " . 
+ "directory '$path' does not exist\n" if ! -d $path; 
+ 
+ cephfs_mount($scfg, $storeid); 
+ } 
+ 
+ $class->SUPER::activate_storage($storeid, $scfg, $cache); 
+} 
+ 
+sub deactivate_storage { 
+ my ($class, $storeid, $scfg, $cache) = @_; 
+ 
+ $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() 
+ if !$cache->{mountdata}; 
+ 
+ my $path = $scfg->{path}; 
+ 
+ if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) { 
+ my $cmd = ['/bin/umount', $path]; 
+ run_command($cmd, errmsg => 'umount error'); 
+ } 
+} 
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile 
index 7b168fa..ad69532 100644 
--- a/PVE/Storage/Makefile 
+++ b/PVE/Storage/Makefile 
@@ -1,4 +1,4 @@ 
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm 
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm 

.PHONY: install 
install: 
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm 
index 163871d..3769e01 100644 
--- a/PVE/Storage/Plugin.pm 
+++ b/PVE/Storage/Plugin.pm 
@@ -24,6 +24,7 @@ our @SHARED_STORAGE = ( 
'nfs', 
'cifs', 
'rbd', 
+ 'cephfs', 
'sheepdog', 
'iscsidirect', 
'glusterfs', 
diff --git a/debian/control b/debian/control 
index 2cf585a..70b5e2d 100644 
--- a/debian/control 
+++ b/debian/control 
@@ -28,6 +28,8 @@ Depends: cstream, 
udev, 
smbclient, 
cifs-utils, 
+ ceph-common, 
+ ceph-fuse, 
${perl:Depends}, 
Description: Proxmox VE storage management library 
This package contains the storage management library used by Proxmox VE. 
-- 
2.11.0 


_______________________________________________ 
pve-devel mailing list 
pve-devel at pve.proxmox.com 
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel 



More information about the pve-devel mailing list