[pve-devel] [PATCH manager v2 3/5] ceph: add CephFS create and list API
Thomas Lamprecht
t.lamprecht at proxmox.com
Thu Nov 22 20:34:20 CET 2018
Allow to create a new CephFS instance and allow to list them.
As deletion requires coordination between the active MDS and all
standby MDS next in line this needs a bit more work. One could mark
the MDS cluster down and stop the active, that should work but as
destroying is quite a sensible operation, in production not often
needed I deemed it better to document this only, and leaving API
endpoints for this to the future.
For index/list I slightly transform the result of an RADOS `fs ls`
monitor command, this would allow relative easy display of a CephFS
and it's backing metadata and data pools in a GUI.
While for now it's not enabled by default and marked as experimental,
this API is designed to host multiple CephFS instances - we may not
need this at all, but I did not want to limit us early. And anybody
liking to experiment can use it after the respective ceph.conf
settings.
When encountering errors try to rollback. As we verified at the
beginning that we did not reused pools, destroy the ones which we
created.
Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
Co-authored-by: Alwin Antreich <a.antreich at proxmox.com>
---
changes v1 -> v2:
* allow to add the newly created CephFS directly to our storage.cfg
* better checks for create, e.g., if any MDS is running (needed immediately for
add_storage and sooner or later else, so always check)
PVE/API2/Ceph.pm | 7 ++
PVE/API2/Ceph/FS.pm | 209 +++++++++++++++++++++++++++++++++++++++++
PVE/API2/Ceph/Makefile | 1 +
PVE/CLI/pveceph.pm | 2 +
PVE/CephTools.pm | 12 +++
5 files changed, 231 insertions(+)
create mode 100644 PVE/API2/Ceph/FS.pm
diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
index d3e8d665..0fc95ab0 100644
--- a/PVE/API2/Ceph.pm
+++ b/PVE/API2/Ceph.pm
@@ -548,6 +548,7 @@ use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::Tools qw(run_command file_get_contents file_set_contents);
+use PVE::API2::Ceph::FS;
use PVE::API2::Ceph::MDS;
use PVE::API2::Storage::Config;
@@ -565,6 +566,11 @@ __PACKAGE__->register_method ({
path => 'mds',
});
+__PACKAGE__->register_method ({
+ subclass => "PVE::API2::Ceph::FS",
+ path => 'fs',
+});
+
__PACKAGE__->register_method ({
name => 'index',
path => '',
@@ -596,6 +602,7 @@ __PACKAGE__->register_method ({
{ name => 'mon' },
{ name => 'osd' },
{ name => 'pools' },
+ { name => 'fs' },
{ name => 'mds' },
{ name => 'stop' },
{ name => 'start' },
diff --git a/PVE/API2/Ceph/FS.pm b/PVE/API2/Ceph/FS.pm
new file mode 100644
index 00000000..2869403b
--- /dev/null
+++ b/PVE/API2/Ceph/FS.pm
@@ -0,0 +1,209 @@
+package PVE::API2::Ceph::FS;
+
+use strict;
+use warnings;
+
+use PVE::CephTools;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RADOS;
+use PVE::RESTHandler;
+use PVE::RPCEnvironment;
+
+use PVE::API2::Storage::Config;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'index',
+ path => '',
+ method => 'GET',
+ description => "Directory index.",
+ permissions => {
+ check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
+ },
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ },
+ },
+ returns => {
+ type => 'array',
+ items => {
+ type => "object",
+ properties => {
+ name => {
+ description => "The ceph filesystem name.",
+ type => 'string',
+ },
+ metadata_pool => {
+ description => "The name of the metadata pool.",
+ type => 'string',
+ },
+ data_pool => {
+ description => "The name of the data pool.",
+ type => 'string',
+ },
+ },
+ },
+ links => [ { rel => 'child', href => "{name}" } ],
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rados = PVE::RADOS->new();
+
+ my $cephfs_list = $rados->mon_command({ prefix => "fs ls" });
+ # we get something like:
+ #{
+ # 'metadata_pool_id' => 2,
+ # 'data_pool_ids' => [ 1 ],
+ # 'metadata_pool' => 'cephfs_metadata',
+ # 'data_pools' => [ 'cephfs_data' ],
+ # 'name' => 'cephfs',
+ #}
+ # pass name for our index and
+
+ my $res = [];
+ map {
+ push @$res, {
+ name => $_->{name},
+ metadata_pool => $_->{metadata_pool},
+ data_pool => $_->{data_pools}->[0],
+ }
+ } @$cephfs_list;
+
+ return $res;
+ }
+});
+
+__PACKAGE__->register_method ({
+ name => 'createfs',
+ path => '{name}',
+ method => 'POST',
+ description => "Create a Ceph filesystem",
+ proxyto => 'node',
+ protected => 1,
+ permissions => {
+ check => ['perm', '/', [ 'Sys.Modify' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ name => {
+ description => "The ceph filesystem name.",
+ type => 'string',
+ default => 'cephfs',
+ optional => 1,
+ },
+ pg_num => {
+ description => "Number of placement groups for the backing data pool. The metadata pool will use a quarter of this.",
+ type => 'integer',
+ default => 64,
+ optional => 1,
+ minimum => 8,
+ maximum => 32768,
+ },
+ add_storage => {
+ description => "Configure the created CephFS as storage for this cluster.",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ },
+ },
+ },
+ returns => { type => 'string' },
+ code => sub {
+ my ($param) = @_;
+
+ PVE::CephTools::check_ceph_inited();
+
+ my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
+ die "Ceph is not fully configured - missing '$pve_ckeyring_path'\n"
+ if ! -f $pve_ckeyring_path;
+
+ my $fs_name = $param->{name} // 'cephfs';
+ my $pg_num = $param->{pg_num} // 64;
+
+ my $pool_data = "${fs_name}_data";
+ my $pool_metadata = "${fs_name}_metadata";
+
+ my $rados = PVE::RADOS->new();
+ my $ls_pools = PVE::CephTools::ls_pools();
+ my $existing_pools = { map { $_->{poolname} => 1 } @$ls_pools };
+
+ die "ceph pools '$pool_data' and/or '$pool_metadata' already exist\n"
+ if $existing_pools->{$pool_data} || $existing_pools->{$pool_metadata};
+
+ my $running_mds = PVE::CephTools::get_cluster_mds_state($rados);
+ die "no running Metadata Server (MDS) found!\n" if !scalar(keys %$running_mds);
+
+ my $worker = sub {
+ $rados = PVE::RADOS->new();
+
+ my $pool_param = {
+ application => 'cephfs',
+ pg_num => $pg_num,
+ };
+
+ my @created_pools = ();
+ eval {
+ print "creating data pool '$pool_data'...\n";
+ PVE::CephTools::create_pool($pool_data, $pool_param, $rados);
+ push @created_pools, $pool_data;
+
+ print "creating metadata pool '$pool_metadata'...\n";
+ $pool_param->{pg_num} = $pg_num >= 32 ? $pg_num / 4 : 8;
+ PVE::CephTools::create_pool($pool_metadata, $pool_param, $rados);
+ push @created_pools, $pool_metadata;
+
+ print "configuring new CephFS '$fs_name'\n";
+ $rados->mon_command({
+ prefix => "fs new",
+ fs_name => $fs_name,
+ metadata => $pool_metadata,
+ data => $pool_data,
+ format => 'plain',
+ });
+ };
+ if (my $err = $@) {
+ $@ = undef;
+
+ if (@created_pools > 0) {
+ warn "Encountered error after creating at least one pool\n";
+ # our old connection is very likely broken now, recreate
+ $rados = PVE::RADOS->new();
+ foreach my $pool (@created_pools) {
+ warn "cleaning up left over pool '$pool'\n";
+ eval { PVE::CephTools::destroy_pool($pool, $rados) };
+ warn "$@\n" if $@;
+ }
+ }
+
+ die "$err\n";
+ }
+
+ if ($param->{add_storage}) {
+ my $err;
+ eval {
+ PVE::API2::Storage::Config->create({
+ type => 'cephfs',
+ storage => $fs_name,
+ content => 'backup,iso,vztmpl',
+ })
+ };
+ die "adding storage for CephFS '$fs_name' failed, check log ".
+ "and add manually!\n$@\n" if $@;
+ }
+ };
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $user = $rpcenv->get_user();
+
+ return $rpcenv->fork_worker('cephfscreate', $fs_name, $user, $worker);
+ }
+});
+
+1;
diff --git a/PVE/API2/Ceph/Makefile b/PVE/API2/Ceph/Makefile
index be4d740c..59fcda71 100644
--- a/PVE/API2/Ceph/Makefile
+++ b/PVE/API2/Ceph/Makefile
@@ -1,6 +1,7 @@
include ../../../defines.mk
PERLSOURCE= \
+ FS.pm \
MDS.pm
all:
diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
index 90878d9e..57097b13 100755
--- a/PVE/CLI/pveceph.pm
+++ b/PVE/CLI/pveceph.pm
@@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
use PVE::JSONSchema qw(get_standard_option);
use PVE::CephTools;
use PVE::API2::Ceph;
+use PVE::API2::Ceph::FS;
use PVE::API2::Ceph::MDS;
use PVE::CLIHandler;
@@ -170,6 +171,7 @@ our $cmddef = {
}],
createpool => [ 'PVE::API2::Ceph', 'createpool', ['name'], { node => $nodename }],
destroypool => [ 'PVE::API2::Ceph', 'destroypool', ['name'], { node => $nodename } ],
+ createfs => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
createosd => [ 'PVE::API2::CephOSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
destroyosd => [ 'PVE::API2::CephOSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename }, $upid_exit],
diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
index da31ccae..80620277 100644
--- a/PVE/CephTools.pm
+++ b/PVE/CephTools.pm
@@ -240,6 +240,18 @@ sub create_pool {
}
+sub ls_pools {
+ my ($pool, $rados) = @_;
+
+ if (!defined($rados)) {
+ $rados = PVE::RADOS->new();
+ }
+
+ my $res = $rados->mon_command({ prefix => "osd lspools" });
+
+ return $res;
+}
+
sub destroy_pool {
my ($pool, $rados) = @_;
--
2.19.1
More information about the pve-devel
mailing list