[pve-devel] [PATCH storage v5 3/3] Cephfs storage plugin

Alwin Antreich a.antreich at proxmox.com
Mon Jun 25 17:50:58 CEST 2018


 - ability to mount through kernel and fuse client
 - allow mount options
 - get MONs from ceph config if not in storage.cfg
 - allow the use of ceph config with fuse client

Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
 PVE/API2/Storage/Config.pm  |  28 ++-----
 PVE/Storage.pm              |   2 +
 PVE/Storage/CephFSPlugin.pm | 177 ++++++++++++++++++++++++++++++++++++++++++++
 PVE/Storage/CephTools.pm    | 102 +++++++++++++++++++++++++
 PVE/Storage/Makefile        |   2 +-
 PVE/Storage/Plugin.pm       |   1 +
 debian/control              |   1 +
 7 files changed, 292 insertions(+), 21 deletions(-)
 create mode 100644 PVE/Storage/CephFSPlugin.pm

diff --git a/PVE/API2/Storage/Config.pm b/PVE/API2/Storage/Config.pm
index 258820d..3abd6e8 100755
--- a/PVE/API2/Storage/Config.pm
+++ b/PVE/API2/Storage/Config.pm
@@ -10,6 +10,7 @@ use PVE::Storage;
 use PVE::Storage::Plugin;
 use PVE::Storage::LVMPlugin;
 use PVE::Storage::CIFSPlugin;
+use PVE::Storage::CephTools;
 use HTTP::Status qw(:constants);
 use Storable qw(dclone);
 use PVE::JSONSchema qw(get_standard_option);
@@ -171,21 +172,10 @@ __PACKAGE__->register_method ({
 		    PVE::Storage::activate_storage($cfg, $baseid);
 
 		    PVE::Storage::LVMPlugin::lvm_create_volume_group($path, $opts->{vgname}, $opts->{shared});
-		} elsif ($type eq 'rbd' && !defined($opts->{monhost})) {
-		    my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring';
-		    my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
-
-		    die "ceph authx keyring file for storage '$storeid' already exists!\n"
-			if -e $ceph_storage_keyring;
-
-		    eval {
-			mkdir '/etc/pve/priv/ceph';
-			PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring);
-		    };
-		    if (my $err = $@) {
-			unlink $ceph_storage_keyring;
-			die "failed to copy ceph authx keyring for storage '$storeid': $err\n";
-		    }
+
+		} elsif (($type eq 'rbd' || $type eq 'cephfs') && !defined($opts->{monhost})) {
+		    # create keyring or secret file in /etc/pve/priv/ceph/
+		    PVE::Storage::CephTools::ceph_create_keyfile($type, $storeid);
 		}
 		# create a password file in /etc/pve/priv,
 		# this file is used as a cert_file at mount time.
@@ -289,11 +279,9 @@ __PACKAGE__->register_method ({
 		    if (-f $cred_file) {
 			unlink($cred_file) or warn "removing cifs credientials '$cred_file' failed: $!\n";
 		    }
-		} elsif ($scfg->{type} eq 'rbd' && !defined($scfg->{monhost})) {
-		    my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
-		    if (-f $ceph_storage_keyring) {
-			unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n";
-		    }
+		} elsif (($scfg->{type} eq 'rbd' || $scfg->{type} eq 'cephfs') && !defined($scfg->{monhost})) {
+		    # remove keyring or secret file in /etc/pve/priv/ceph/
+		    PVE::Storage::CephTools::ceph_remove_keyfile($scfg->{type}, $storeid);
 		}
 
 		delete $cfg->{ids}->{$storeid};
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index d733380..f9732fe 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin;
 use PVE::Storage::CIFSPlugin;
 use PVE::Storage::ISCSIPlugin;
 use PVE::Storage::RBDPlugin;
+use PVE::Storage::CephFSPlugin;
 use PVE::Storage::SheepdogPlugin;
 use PVE::Storage::ISCSIDirectPlugin;
 use PVE::Storage::GlusterfsPlugin;
@@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register();
 PVE::Storage::CIFSPlugin->register();
 PVE::Storage::ISCSIPlugin->register();
 PVE::Storage::RBDPlugin->register();
+PVE::Storage::CephFSPlugin->register();
 PVE::Storage::SheepdogPlugin->register();
 PVE::Storage::ISCSIDirectPlugin->register();
 PVE::Storage::GlusterfsPlugin->register();
diff --git a/PVE/Storage/CephFSPlugin.pm b/PVE/Storage/CephFSPlugin.pm
new file mode 100644
index 0000000..fa687f4
--- /dev/null
+++ b/PVE/Storage/CephFSPlugin.pm
@@ -0,0 +1,177 @@
+package PVE::Storage::CephFSPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use Net::IP;
+use File::Path;
+use PVE::Tools qw(run_command);
+use PVE::ProcFSTools;
+use PVE::Storage::Plugin;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::Storage::CephTools;
+
+use base qw(PVE::Storage::Plugin);
+
+sub cephfs_is_mounted {
+    my ($scfg, $storeid, $mountdata) = @_;
+
+    my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid);
+    my $configfile = $cmd_option->{ceph_conf};
+    my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile);
+
+    my $subdir = $scfg->{subdir} // '/';
+    my $mountpoint = $scfg->{path};
+    my $source = "$server:$subdir";
+
+    $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
+    return $mountpoint if grep {
+	$_->[2] =~ m#^ceph|fuse\.ceph-fuse# &&
+	$_->[0] =~ m#^\Q$source\E|ceph-fuse$# &&
+	$_->[1] eq $mountpoint
+    } @$mountdata;
+
+    warn "A filesystem is already mounted on $mountpoint\n"
+	if grep { $_->[1] eq $mountpoint } @$mountdata;
+
+    return undef;
+}
+
+sub cephfs_mount {
+    my ($scfg, $storeid) = @_;
+
+    my $cmd;
+    my $mountpoint = $scfg->{path};
+    my $subdir = $scfg->{subdir} // '/';
+
+    my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid);
+    my $configfile = $cmd_option->{ceph_conf};
+    my $secretfile = $cmd_option->{keyring};
+    my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile);
+
+    # fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg
+    # kernel -> better performance, less frequent updates
+    if ($scfg->{fuse}) {
+	    # FIXME: ceph-fuse client complains about missing ceph.conf or keyring if
+	    # not provided on its default locations but still connects. Fix upstream??
+	    $cmd = ['/usr/bin/ceph-fuse', '-n', "client.$cmd_option->{userid}", '-m', $server];
+	    push @$cmd, '--keyfile', $secretfile if defined($secretfile);
+	    push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|);
+	    push @$cmd, $mountpoint;
+	    push @$cmd, '--conf', $configfile if defined($configfile);
+    }else {
+	my $source = "$server:$subdir";
+	$cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', "name=$cmd_option->{userid}"];
+	push @$cmd, '-o', "secretfile=$secretfile" if defined($secretfile);
+    }
+
+    if ($scfg->{options}) {
+	push @$cmd, '-o', $scfg->{options};
+    }
+
+    run_command($cmd, errmsg => "mount error");
+}
+
+# Configuration
+
+sub type {
+    return 'cephfs';
+}
+
+sub plugindata {
+    return {
+	content => [ { vztmpl => 1, iso => 1, backup => 1},
+		     { backup => 1 }],
+    };
+}
+
+sub properties {
+    return {
+	fuse => {
+	    description => "Mount CephFS through FUSE.",
+	    type => 'boolean',
+	},
+	subdir => {
+	    description => "Subdir to mount.",
+	    type => 'string', format => 'pve-storage-path',
+	},
+    };
+}
+
+sub options {
+    return {
+	path => { fixed => 1 },
+	monhost => { optional => 1},
+	nodes => { optional => 1 },
+	subdir => { optional => 1 },
+	disable => { optional => 1 },
+	options => { optional => 1 },
+	username => { optional => 1 },
+	content => { optional => 1 },
+	format => { optional => 1 },
+	mkdir => { optional => 1 },
+	fuse => { optional => 1 },
+	bwlimit => { optional => 1 },
+    };
+}
+
+sub check_config {
+    my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_;
+
+    $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path};
+
+    return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck);
+}
+
+# Storage implementation
+
+sub status {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+	if !$cache->{mountdata};
+
+    return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata});
+
+    return $class->SUPER::status($storeid, $scfg, $cache);
+}
+
+sub activate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+	if !$cache->{mountdata};
+
+    my $path = $scfg->{path};
+
+    if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
+
+	# NOTE: only call mkpath when not mounted (avoid hang
+	# when cephfs is offline
+
+	mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir});
+
+	die "unable to activate storage '$storeid' - " .
+	    "directory '$path' does not exist\n" if ! -d $path;
+
+	cephfs_mount($scfg, $storeid);
+    }
+
+    $class->SUPER::activate_storage($storeid, $scfg, $cache);
+}
+
+sub deactivate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+	if !$cache->{mountdata};
+
+    my $path = $scfg->{path};
+
+    if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
+	my $cmd = ['/bin/umount', $path];
+	run_command($cmd, errmsg => 'umount error');
+    }
+}
+
+1;
diff --git a/PVE/Storage/CephTools.pm b/PVE/Storage/CephTools.pm
index 3e2cede..766163d 100644
--- a/PVE/Storage/CephTools.pm
+++ b/PVE/Storage/CephTools.pm
@@ -34,6 +34,64 @@ my $ceph_check_keyfile = sub {
     return undef;
 };
 
+my $parse_ceph_file = sub {
+    my ($filename) = @_;
+
+    my $cfg = {};
+
+    return $cfg if ! -f $filename;
+
+    my $content = PVE::Tools::file_get_contents($filename);
+    my @lines = split /\n/, $content;
+
+    my $section;
+
+    foreach my $line (@lines) {
+	$line =~ s/[;#].*$//;
+	$line =~ s/^\s+//;
+	$line =~ s/\s+$//;
+	next if !$line;
+
+	$section = $1 if $line =~ m/^\[(\S+)\]$/;
+	if (!$section) {
+	    warn "no section - skip: $line\n";
+	    next;
+	}
+
+	if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
+	    $cfg->{$section}->{$1} = $2;
+	}
+
+    }
+
+    return $cfg;
+};
+
+my $ceph_get_key = sub {
+    my ($keyfile, $username) = @_;
+
+    my $key = $parse_ceph_file->($keyfile);
+    my $secret = $key->{"client.$username"}->{key};
+
+    return $secret;
+};
+
+sub get_monaddr_list {
+    my ($configfile) = shift;
+
+    my $server;
+
+    if (!defined($configfile)) {
+	warn "No ceph config specified\n";
+	return;
+    }
+
+    my $config = $parse_ceph_file->($configfile);
+    @$server = sort map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config};
+
+    return join(',', @$server);
+};
+
 sub hostlist {
     my ($list_text, $separator) = @_;
 
@@ -85,4 +143,48 @@ sub ceph_connect_option {
 
 }
 
+sub ceph_create_keyfile {
+    my ($type, $storeid) = @_;
+
+    my $extension = 'keyring';
+    $extension = 'secret' if ($type eq 'cephfs');
+
+    my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring';
+    my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension";
+
+    die "ceph authx keyring file for storage '$storeid' already exists!\n"
+	if -e $ceph_storage_keyring;
+
+    if (-e $ceph_admin_keyring) {
+	eval {
+	    if ($type eq 'rbd') {
+		mkdir '/etc/pve/priv/ceph';
+		PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring);
+	    } elsif ($type eq 'cephfs') {
+		my $secret = $ceph_get_key->($ceph_admin_keyring, 'admin');
+		mkdir '/etc/pve/priv/ceph';
+		PVE::Tools::file_set_contents($ceph_storage_keyring, $secret, 0400);
+	   }
+	};
+	if (my $err = $@) {
+	   unlink $ceph_storage_keyring;
+	   die "failed to copy ceph authx $extension for storage '$storeid': $err\n";
+	}
+    } else {
+	warn "$ceph_admin_keyring not found, authentication is disabled.\n";
+    }
+}
+
+sub ceph_remove_keyfile {
+    my ($type, $storeid) = @_;
+
+    my $extension = 'keyring';
+    $extension = 'secret' if ($type eq 'cephfs');
+    my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension";
+
+    if (-f $ceph_storage_keyring) {
+	unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n";
+    }
+}
+
 1;
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index 82dadd6..c7f423f 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
 
 .PHONY: install
 install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 163871d..3769e01 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -24,6 +24,7 @@ our @SHARED_STORAGE = (
     'nfs',
     'cifs',
     'rbd',
+    'cephfs',
     'sheepdog',
     'iscsidirect',
     'glusterfs',
diff --git a/debian/control b/debian/control
index 908dd24..9923350 100644
--- a/debian/control
+++ b/debian/control
@@ -28,6 +28,7 @@ Depends: cstream,
          udev,
          smbclient,
          ceph-common,
+         ceph-fuse,
          cifs-utils,
          ${perl:Depends},
 Description: Proxmox VE storage management library
-- 
2.11.0




More information about the pve-devel mailing list