[pve-devel] [PATCH manager v2 3/5] ceph: add CephFS create and list API
Dominik Csapak
d.csapak at proxmox.com
Fri Nov 23 11:02:23 CET 2018
one small comment inline, looks good otherwise
On 11/22/18 8:34 PM, Thomas Lamprecht wrote:
> Allow to create a new CephFS instance and allow to list them.
>
> As deletion requires coordination between the active MDS and all
> standby MDS next in line this needs a bit more work. One could mark
> the MDS cluster down and stop the active, that should work but as
> destroying is quite a sensible operation, in production not often
> needed I deemed it better to document this only, and leaving API
> endpoints for this to the future.
>
> For index/list I slightly transform the result of an RADOS `fs ls`
> monitor command, this would allow relative easy display of a CephFS
> and it's backing metadata and data pools in a GUI.
>
> While for now it's not enabled by default and marked as experimental,
> this API is designed to host multiple CephFS instances - we may not
> need this at all, but I did not want to limit us early. And anybody
> liking to experiment can use it after the respective ceph.conf
> settings.
>
> When encountering errors try to rollback. As we verified at the
> beginning that we did not reused pools, destroy the ones which we
> created.
>
> Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
> Co-authored-by: Alwin Antreich <a.antreich at proxmox.com>
> ---
>
> changes v1 -> v2:
> * allow to add the newly created CephFS directly to our storage.cfg
> * better checks for create, e.g., if any MDS is running (needed immediately for
> add_storage and sooner or later else, so always check)
>
> PVE/API2/Ceph.pm | 7 ++
> PVE/API2/Ceph/FS.pm | 209 +++++++++++++++++++++++++++++++++++++++++
> PVE/API2/Ceph/Makefile | 1 +
> PVE/CLI/pveceph.pm | 2 +
> PVE/CephTools.pm | 12 +++
> 5 files changed, 231 insertions(+)
> create mode 100644 PVE/API2/Ceph/FS.pm
>
> diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
> index d3e8d665..0fc95ab0 100644
> --- a/PVE/API2/Ceph.pm
> +++ b/PVE/API2/Ceph.pm
> @@ -548,6 +548,7 @@ use PVE::RPCEnvironment;
> use PVE::Storage;
> use PVE::Tools qw(run_command file_get_contents file_set_contents);
>
> +use PVE::API2::Ceph::FS;
> use PVE::API2::Ceph::MDS;
> use PVE::API2::Storage::Config;
>
> @@ -565,6 +566,11 @@ __PACKAGE__->register_method ({
> path => 'mds',
> });
>
> +__PACKAGE__->register_method ({
> + subclass => "PVE::API2::Ceph::FS",
> + path => 'fs',
> +});
> +
> __PACKAGE__->register_method ({
> name => 'index',
> path => '',
> @@ -596,6 +602,7 @@ __PACKAGE__->register_method ({
> { name => 'mon' },
> { name => 'osd' },
> { name => 'pools' },
> + { name => 'fs' },
> { name => 'mds' },
> { name => 'stop' },
> { name => 'start' },
> diff --git a/PVE/API2/Ceph/FS.pm b/PVE/API2/Ceph/FS.pm
> new file mode 100644
> index 00000000..2869403b
> --- /dev/null
> +++ b/PVE/API2/Ceph/FS.pm
> @@ -0,0 +1,209 @@
> +package PVE::API2::Ceph::FS;
> +
> +use strict;
> +use warnings;
> +
> +use PVE::CephTools;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::RADOS;
> +use PVE::RESTHandler;
> +use PVE::RPCEnvironment;
> +
> +use PVE::API2::Storage::Config;
> +
> +use base qw(PVE::RESTHandler);
> +
> +__PACKAGE__->register_method ({
> + name => 'index',
> + path => '',
> + method => 'GET',
> + description => "Directory index.",
> + permissions => {
> + check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
> + },
> + protected => 1,
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + },
> + },
> + returns => {
> + type => 'array',
> + items => {
> + type => "object",
> + properties => {
> + name => {
> + description => "The ceph filesystem name.",
> + type => 'string',
> + },
> + metadata_pool => {
> + description => "The name of the metadata pool.",
> + type => 'string',
> + },
> + data_pool => {
> + description => "The name of the data pool.",
> + type => 'string',
> + },
> + },
> + },
> + links => [ { rel => 'child', href => "{name}" } ],
> + },
> + code => sub {
> + my ($param) = @_;
> +
> + my $rados = PVE::RADOS->new();
> +
> + my $cephfs_list = $rados->mon_command({ prefix => "fs ls" });
> + # we get something like:
> + #{
> + # 'metadata_pool_id' => 2,
> + # 'data_pool_ids' => [ 1 ],
> + # 'metadata_pool' => 'cephfs_metadata',
> + # 'data_pools' => [ 'cephfs_data' ],
> + # 'name' => 'cephfs',
> + #}
> + # pass name for our index and
looks like there was more of the comment?
> +
> + my $res = [];
> + map {
> + push @$res, {
> + name => $_->{name},
> + metadata_pool => $_->{metadata_pool},
> + data_pool => $_->{data_pools}->[0],
> + }
> + } @$cephfs_list;
> +
> + return $res;
> + }
> +});
> +
> +__PACKAGE__->register_method ({
> + name => 'createfs',
> + path => '{name}',
> + method => 'POST',
> + description => "Create a Ceph filesystem",
> + proxyto => 'node',
> + protected => 1,
> + permissions => {
> + check => ['perm', '/', [ 'Sys.Modify' ]],
> + },
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + name => {
> + description => "The ceph filesystem name.",
> + type => 'string',
> + default => 'cephfs',
> + optional => 1,
> + },
> + pg_num => {
> + description => "Number of placement groups for the backing data pool. The metadata pool will use a quarter of this.",
> + type => 'integer',
> + default => 64,
> + optional => 1,
> + minimum => 8,
> + maximum => 32768,
> + },
> + add_storage => {
> + description => "Configure the created CephFS as storage for this cluster.",
> + type => 'boolean',
> + optional => 1,
> + default => 0,
> + },
> + },
> + },
> + returns => { type => 'string' },
> + code => sub {
> + my ($param) = @_;
> +
> + PVE::CephTools::check_ceph_inited();
> +
> + my $pve_ckeyring_path = PVE::CephTools::get_config('pve_ckeyring_path');
> + die "Ceph is not fully configured - missing '$pve_ckeyring_path'\n"
> + if ! -f $pve_ckeyring_path;
> +
> + my $fs_name = $param->{name} // 'cephfs';
> + my $pg_num = $param->{pg_num} // 64;
> +
> + my $pool_data = "${fs_name}_data";
> + my $pool_metadata = "${fs_name}_metadata";
> +
> + my $rados = PVE::RADOS->new();
> + my $ls_pools = PVE::CephTools::ls_pools();
> + my $existing_pools = { map { $_->{poolname} => 1 } @$ls_pools };
> +
> + die "ceph pools '$pool_data' and/or '$pool_metadata' already exist\n"
> + if $existing_pools->{$pool_data} || $existing_pools->{$pool_metadata};
> +
> + my $running_mds = PVE::CephTools::get_cluster_mds_state($rados);
> + die "no running Metadata Server (MDS) found!\n" if !scalar(keys %$running_mds);
> +
> + my $worker = sub {
> + $rados = PVE::RADOS->new(); > +
> + my $pool_param = {
> + application => 'cephfs',
> + pg_num => $pg_num,
> + };
> +
> + my @created_pools = ();
> + eval {
> + print "creating data pool '$pool_data'...\n";
> + PVE::CephTools::create_pool($pool_data, $pool_param, $rados);
> + push @created_pools, $pool_data;
> +
> + print "creating metadata pool '$pool_metadata'...\n";
> + $pool_param->{pg_num} = $pg_num >= 32 ? $pg_num / 4 : 8;
> + PVE::CephTools::create_pool($pool_metadata, $pool_param, $rados);
> + push @created_pools, $pool_metadata;
> +
> + print "configuring new CephFS '$fs_name'\n";
> + $rados->mon_command({
> + prefix => "fs new",
> + fs_name => $fs_name,
> + metadata => $pool_metadata,
> + data => $pool_data,
> + format => 'plain',
> + });
> + };
> + if (my $err = $@) {
> + $@ = undef;
> +
> + if (@created_pools > 0) {
> + warn "Encountered error after creating at least one pool\n";
> + # our old connection is very likely broken now, recreate
> + $rados = PVE::RADOS->new();
> + foreach my $pool (@created_pools) {
> + warn "cleaning up left over pool '$pool'\n";
> + eval { PVE::CephTools::destroy_pool($pool, $rados) };
> + warn "$@\n" if $@;
> + }
> + }
> +
> + die "$err\n";
> + }
> +
> + if ($param->{add_storage}) {
> + my $err;
> + eval {
> + PVE::API2::Storage::Config->create({
> + type => 'cephfs',
> + storage => $fs_name,
> + content => 'backup,iso,vztmpl',
> + })
> + };
> + die "adding storage for CephFS '$fs_name' failed, check log ".
> + "and add manually!\n$@\n" if $@;
> + }
> + };
> +
> + my $rpcenv = PVE::RPCEnvironment::get();
> + my $user = $rpcenv->get_user();
> +
> + return $rpcenv->fork_worker('cephfscreate', $fs_name, $user, $worker);
> + }
> +});
> +
> +1;
> diff --git a/PVE/API2/Ceph/Makefile b/PVE/API2/Ceph/Makefile
> index be4d740c..59fcda71 100644
> --- a/PVE/API2/Ceph/Makefile
> +++ b/PVE/API2/Ceph/Makefile
> @@ -1,6 +1,7 @@
> include ../../../defines.mk
>
> PERLSOURCE= \
> + FS.pm \
> MDS.pm
>
> all:
> diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
> index 90878d9e..57097b13 100755
> --- a/PVE/CLI/pveceph.pm
> +++ b/PVE/CLI/pveceph.pm
> @@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
> use PVE::JSONSchema qw(get_standard_option);
> use PVE::CephTools;
> use PVE::API2::Ceph;
> +use PVE::API2::Ceph::FS;
> use PVE::API2::Ceph::MDS;
>
> use PVE::CLIHandler;
> @@ -170,6 +171,7 @@ our $cmddef = {
> }],
> createpool => [ 'PVE::API2::Ceph', 'createpool', ['name'], { node => $nodename }],
> destroypool => [ 'PVE::API2::Ceph', 'destroypool', ['name'], { node => $nodename } ],
> + createfs => [ 'PVE::API2::Ceph::FS', 'createfs', [], { node => $nodename }],
> createosd => [ 'PVE::API2::CephOSD', 'createosd', ['dev'], { node => $nodename }, $upid_exit],
> destroyosd => [ 'PVE::API2::CephOSD', 'destroyosd', ['osdid'], { node => $nodename }, $upid_exit],
> createmon => [ 'PVE::API2::Ceph', 'createmon', [], { node => $nodename }, $upid_exit],
> diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
> index da31ccae..80620277 100644
> --- a/PVE/CephTools.pm
> +++ b/PVE/CephTools.pm
> @@ -240,6 +240,18 @@ sub create_pool {
>
> }
>
> +sub ls_pools {
> + my ($pool, $rados) = @_;
> +
> + if (!defined($rados)) {
> + $rados = PVE::RADOS->new();
> + }
> +
> + my $res = $rados->mon_command({ prefix => "osd lspools" });
> +
> + return $res;
> +}
> +
> sub destroy_pool {
> my ($pool, $rados) = @_;
>
>
More information about the pve-devel
mailing list