[pve-devel] [PATCH manager 6/8] ceph: add MDS create/delete/list API
Thomas Lamprecht
t.lamprecht at proxmox.com
Wed Nov 21 11:42:41 CET 2018
Allow to create, list and destroy and Ceph Metadata Server (MDS) over
the API and the CLI `pveceph` tool.
Besides setting up the local systemd service template and the MDS
data directory we also add a reference to the MDS in the ceph.conf
We note the backing host (node) from the respective MDS and set up a
'mds standby for name' = 'pve' so that the PVE created ones are a
single group. If we decide to add integration for rank/path specific
MDS (possible useful for CephFS with quite a bit of load) then this
may help as a starting point.
On create, check early if a reference already exists in ceph.conf and
abort in that case. If we only see existing data directories later
on we do not remove them, they could well be from an older manual
create - where it's possible dangerous to just remove it. Let the
user handle it themself in that case.
Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
Co-authored-by: Alwin Antreich <a.antreich at proxmox.com>
---
Not fully sure about the 'mds standby for name' so if you have objections
regarding (negative) implications of this we can remove it just fine.
PVE/API2/Ceph.pm | 7 ++
PVE/API2/Ceph/MDS.pm | 196 +++++++++++++++++++++++++++++++++++++++++
PVE/API2/Ceph/Makefile | 15 ++++
PVE/API2/Makefile | 4 +
PVE/CLI/pveceph.pm | 3 +
PVE/CephTools.pm | 99 ++++++++++++++++++++-
6 files changed, 323 insertions(+), 1 deletion(-)
create mode 100644 PVE/API2/Ceph/MDS.pm
create mode 100644 PVE/API2/Ceph/Makefile
diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
index 854527a3..a8665dc1 100644
--- a/PVE/API2/Ceph.pm
+++ b/PVE/API2/Ceph.pm
@@ -542,6 +542,7 @@ use PVE::Storage;
use PVE::RESTHandler;
use PVE::RPCEnvironment;
use PVE::JSONSchema qw(get_standard_option);
+use PVE::API2::Ceph::MDS;
use PVE::RADOS;
use PVE::CephTools;
use PVE::Network;
@@ -555,6 +556,11 @@ __PACKAGE__->register_method ({
path => 'osd',
});
+__PACKAGE__->register_method ({
+ subclass => "PVE::API2::Ceph::MDS",
+ path => 'mds',
+});
+
__PACKAGE__->register_method ({
name => 'index',
path => '',
@@ -586,6 +592,7 @@ __PACKAGE__->register_method ({
{ name => 'mon' },
{ name => 'osd' },
{ name => 'pools' },
+ { name => 'mds' },
{ name => 'stop' },
{ name => 'start' },
{ name => 'status' },
diff --git a/PVE/API2/Ceph/MDS.pm b/PVE/API2/Ceph/MDS.pm
new file mode 100644
index 00000000..990fd0fc
--- /dev/null
+++ b/PVE/API2/Ceph/MDS.pm
@@ -0,0 +1,196 @@
+package PVE::API2::Ceph::MDS;
+
+use strict;
+use warnings;
+
+use PVE::CephTools;
+use PVE::API2::Ceph;
+use PVE::API2::Storage::Config;
+use PVE::RPCEnvironment;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RADOS;
+use PVE::RESTHandler;
+use PVE::INotify;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'index',
+ path => '',
+ method => 'GET',
+ description => "Directory index.",
+ permissions => {
+ check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => "object",
+ properties => {},
+ },
+ links => [ { rel => 'child', href => "{id}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $mds_list = PVE::CephTools::list_local_mds_ids();
+
+ my $res = [];
+ foreach my $mds (@$mds_list) {
+ push @$res, { 'id' => $mds };
+ }
+
+ return $res;
+ }
+});
+
+__PACKAGE__->register_method ({
+ name => 'createmds',
+ path => '{id}',
+ method => 'POST',
+ description => "Create Ceph Metadata Server (MDS)",
+ proxyto => 'node',
+ protected => 1,
+ permissions => {
+ check => ['perm', '/', [ 'Sys.Modify' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ id => {
+ type => 'string',
+ optional => 1,
+ default => 'nodename',
+ pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
+ description => "The ID for the mds, when omitted the same as the nodename",
+ },
+ hotstandby => {
+ type => 'boolean',
+ optional => 1,
+ default => '0',
+ description => "Determines whether a ceph-mds daemon should poll and replay the log of an active MDS. ".
+ "Faster switch on MDS failure, but needs more idle resources.",
+ },
+ },
+ },
+ returns => { type => 'string' },
+ code => sub {
+ my ($param) = @_;
+
+ PVE::CephTools::check_ceph_installed('ceph_mds');
+
+ PVE::CephTools::check_ceph_inited();
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $nodename = $param->{node};
+ $nodename = INotify::nodename() if $nodename eq 'localhost';
+
+ my $mds_id = $param->{id} // $nodename;
+
+ my $worker = sub {
+ my $timeout = PVE::CephTools::get_config('long_rados_timeout');
+ my $rados = PVE::RADOS->new(timeout => $timeout);
+
+ my $cfg = PVE::CephTools::parse_ceph_config();
+
+ my $section = "mds.$mds_id";
+
+ if (defined($cfg->{$section})) {
+ die "MDS '$mds_id' already referenced in ceph config, abort!\n"
+ }
+
+ if (!defined($cfg->{mds}->{keyring})) {
+ # $id isn't a perl variable but a ceph metavariable
+ my $keyring = '/var/lib/ceph/mds/ceph-$id/keyring';
+
+ $cfg->{mds}->{keyring} = $keyring;
+ }
+
+ $cfg->{$section}->{host} = $nodename;
+ $cfg->{$section}->{"mds standby for name"} = 'pve';
+
+ if ($param->{hotstandby}) {
+ $cfg->{$section}->{"mds standby replay"} = 'true';
+ }
+
+ PVE::CephTools::write_ceph_config($cfg);
+
+ eval { PVE::CephTools::create_mds($mds_id, $rados) };
+ if (my $err = $@) {
+ # we abort early if the section is defined, so we know that we
+ # wrote it at this point. Do not auto remove the service, could
+ # do real harm for previously manual setup MDS
+ warn "Encountered error, remove '$section' from ceph.conf\n";
+ $cfg = PVE::CephTools::parse_ceph_config();
+ delete $cfg->{$section};
+ PVE::CephTools::write_ceph_config($cfg);
+
+ die "$err\n";
+ }
+ };
+
+ return $rpcenv->fork_worker('cephcreatemds', "mds.$mds_id", $authuser, $worker);
+ }
+});
+
+__PACKAGE__->register_method ({
+ name => 'destroymds',
+ path => '{id}',
+ method => 'DELETE',
+ description => "Destroy Ceph Metadata Server",
+ proxyto => 'node',
+ protected => 1,
+ permissions => {
+ check => ['perm', '/', [ 'Sys.Modify' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ id => {
+ description => 'The ID of the mds',
+ type => 'string',
+ pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
+ },
+ },
+ },
+ returns => { type => 'string' },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+
+ my $authuser = $rpcenv->get_user();
+
+ PVE::CephTools::check_ceph_inited();
+
+ my $mds_id = $param->{id};
+
+ my $worker = sub {
+ my $timeout = PVE::CephTools::get_config('long_rados_timeout');
+ my $rados = PVE::RADOS->new(timeout => $timeout);
+
+ my $cfg = PVE::CephTools::parse_ceph_config();
+
+ if (defined($cfg->{"mds.$mds_id"})) {
+ delete $cfg->{"mds.$mds_id"};
+ PVE::CephTools::write_ceph_config($cfg);
+ }
+
+ PVE::CephTools::destroy_mds($mds_id, $rados);
+ };
+
+ return $rpcenv->fork_worker('cephdestroymds', "mds.$mds_id", $authuser, $worker);
+ }
+});
+
+1;
diff --git a/PVE/API2/Ceph/Makefile b/PVE/API2/Ceph/Makefile
new file mode 100644
index 00000000..be4d740c
--- /dev/null
+++ b/PVE/API2/Ceph/Makefile
@@ -0,0 +1,15 @@
+include ../../../defines.mk
+
+PERLSOURCE= \
+ MDS.pm
+
+all:
+
+.PHONY: clean
+clean:
+ rm -rf *~
+
+.PHONY: install
+install: ${PERLSOURCE}
+ install -d ${PERLLIBDIR}/PVE/API2/Ceph
+ install -m 0644 ${PERLSOURCE} ${PERLLIBDIR}/PVE/API2/Ceph
diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile
index 75b022dd..20649ea5 100644
--- a/PVE/API2/Makefile
+++ b/PVE/API2/Makefile
@@ -1,5 +1,7 @@
include ../../defines.mk
+SUBDIRS=Ceph
+
PERLSOURCE = \
Replication.pm \
ReplicationConfig.pm \
@@ -26,8 +28,10 @@ all:
.PHONY: clean
clean:
rm -rf *~
+ set -e && for i in ${SUBDIRS}; do ${MAKE} -C $$i $@; done
.PHONY: install
install: ${PERLSOURCE}
install -d ${PERLLIBDIR}/PVE/API2
install -m 0644 ${PERLSOURCE} ${PERLLIBDIR}/PVE/API2
+ set -e && for i in ${SUBDIRS}; do ${MAKE} -C $$i $@; done
diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
index a5a04949..90878d9e 100755
--- a/PVE/CLI/pveceph.pm
+++ b/PVE/CLI/pveceph.pm
@@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
use PVE::JSONSchema qw(get_standard_option);
use PVE::CephTools;
use PVE::API2::Ceph;
+use PVE::API2::Ceph::MDS;
use PVE::CLIHandler;
@@ -175,6 +176,8 @@ our $cmddef = {
destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
createmgr => [ 'PVE::API2::Ceph', 'createmgr', [], { node => $nodename }, $upid_exit],
destroymgr => [ 'PVE::API2::Ceph', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
+ createmds => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
+ destroymds => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['id'], { node => $nodename }, $upid_exit],
start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, $upid_exit],
stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, $upid_exit],
install => [ __PACKAGE__, 'install', [] ],
diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
index 93d531b6..d0e774ed 100644
--- a/PVE/CephTools.pm
+++ b/PVE/CephTools.pm
@@ -17,12 +17,14 @@ my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
+my $ceph_mds_data_dir = '/var/lib/ceph/mds';
my $ceph_service = {
ceph_bin => "/usr/bin/ceph",
ceph_mon => "/usr/bin/ceph-mon",
ceph_mgr => "/usr/bin/ceph-mgr",
- ceph_osd => "/usr/bin/ceph-osd"
+ ceph_osd => "/usr/bin/ceph-osd",
+ ceph_mds => "/usr/bin/ceph-mds",
};
my $config_hash = {
@@ -32,6 +34,7 @@ my $config_hash = {
pve_ckeyring_path => $pve_ckeyring_path,
ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
+ ceph_mds_data_dir => $ceph_mds_data_dir,
long_rados_timeout => 60,
};
@@ -296,4 +299,98 @@ sub systemd_managed {
}
}
+sub list_local_mds_ids {
+ my $mds_list = [];
+
+ PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
+ my (undef, $mds_id) = @_;
+ push @$mds_list, $mds_id;
+ });
+
+ return $mds_list;
+}
+
+sub create_mds {
+ my ($id, $rados) = @_;
+
+ # `ceph fs status` fails with numeric only ID.
+ die "ID: $id, numeric only IDs are not supported\n"
+ if $id =~ /^\d+$/;
+
+ if (!defined($rados)) {
+ $rados = PVE::RADOS->new();
+ }
+
+ my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
+ my $service_keyring = "$service_dir/keyring";
+ my $service_name = "mds.$id";
+
+ die "ceph MDS directory '$service_dir' already exists\n"
+ if -d $service_dir;
+
+ print "creating MDS directory '$service_dir'\n";
+ eval { File::Path::mkpath($service_dir) };
+ my $err = $@;
+ die "creation MDS directory '$service_dir' failed\n" if $err;
+
+ # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
+ my $priv = [
+ mon => 'allow profile mds',
+ osd => 'allow rwx',
+ mds => 'allow *',
+ ];
+
+ print "creating keys for '$service_name'\n";
+ my $output = $rados->mon_command({
+ prefix => 'auth get-or-create',
+ entity => $service_name,
+ caps => $priv,
+ format => 'plain',
+ });
+
+ PVE::Tools::file_set_contents($service_keyring, $output);
+
+ print "setting ceph as owner for service directory\n";
+ run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
+
+ print "enabling service 'ceph-mds\@$id.service'\n";
+ ceph_service_cmd('enable', $service_name);
+ print "starting service 'ceph-mds\@$id.service'\n";
+ ceph_service_cmd('start', $service_name);
+
+ return undef;
+};
+
+sub destroy_mds {
+ my ($id, $rados) = @_;
+
+ if (!defined($rados)) {
+ $rados = PVE::RADOS->new();
+ }
+
+ my $service_name = "mds.$id";
+ my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
+
+ print "disabling service 'ceph-mds\@$id.service'\n";
+ ceph_service_cmd('disable', $service_name);
+ print "stopping service 'ceph-mds\@$id.service'\n";
+ ceph_service_cmd('stop', $service_name);
+
+ if (-d $service_dir) {
+ print "removing ceph-mds directory '$service_dir'\n";
+ File::Path::remove_tree($service_dir);
+ } else {
+ warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
+ }
+
+ print "removing ceph auth for '$service_name'\n";
+ $rados->mon_command({
+ prefix => 'auth del',
+ entity => $service_name,
+ format => 'plain'
+ });
+
+ return undef;
+};
+
1;
--
2.19.1
More information about the pve-devel
mailing list