[pve-devel] [PATCH manager 1/2] Create/destroy Ceph MDS

[Thomas Lamprecht]

On 11/14/18 1:23 PM, Alwin Antreich wrote:
> * introduce generic ceph service create/destroy methods
> * add API for createmds/destroymds
> * add createmds/destroymds to pveceph CLI
> 
> Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
> ---
>  PVE/API2/Makefile  |   1 +
>  PVE/API2/Ceph.pm   |   6 +++
>  PVE/API2/CephFS.pm | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  PVE/CLI/pveceph.pm |   3 ++
>  PVE/CephTools.pm   |  73 ++++++++++++++++++++++++++++-
>  5 files changed, 213 insertions(+), 1 deletion(-)
>  create mode 100644 PVE/API2/CephFS.pm
> 
> diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile
> index 9862e498..29ffac01 100644
> --- a/PVE/API2/Makefile
> +++ b/PVE/API2/Makefile
> @@ -4,6 +4,7 @@ PERLSOURCE = 			\
>  	Replication.pm		\
>  	ReplicationConfig.pm	\
>  	Ceph.pm			\
> +	CephFS.pm		\
>  	APT.pm			\
>  	Subscription.pm		\
>  	VZDump.pm		\
> diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
> index 8584cb51..20525c78 100644
> --- a/PVE/API2/Ceph.pm
> +++ b/PVE/API2/Ceph.pm
> @@ -575,6 +575,11 @@ __PACKAGE__->register_method ({
>  });
>  
>  __PACKAGE__->register_method ({
> +    subclass => "PVE::API2::CephFS",

you also need to use this new module to allow this to work.
CephOSD is in the same file here, so it doesn't need that use,
if you wonder why you need to do it differently that the existing
CephOSD where you based your changes on.

This would fix your 'no such resource' error

> +    path => 'fs',
> +});
> +
> +__PACKAGE__->register_method ({
>      name => 'index',
>      path => '',
>      method => 'GET',
> @@ -605,6 +610,7 @@ __PACKAGE__->register_method ({
>  	    { name => 'mon' },
>  	    { name => 'osd' },
>  	    { name => 'pools' },
> +	    { name => 'fs' },
>  	    { name => 'stop' },
>  	    { name => 'start' },
>  	    { name => 'status' },
> diff --git a/PVE/API2/CephFS.pm b/PVE/API2/CephFS.pm
> new file mode 100644
> index 00000000..b1c32b4f
> --- /dev/null
> +++ b/PVE/API2/CephFS.pm
> @@ -0,0 +1,131 @@
> +package PVE::API2::CephFS;
> +
> +use strict;
> +use warnings;
> +use Cwd qw(abs_path);
> +use PVE::CephTools;
> +use PVE::RPCEnvironment;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::RADOS;
> +use PVE::RESTHandler;
> +
> +use base qw(PVE::RESTHandler);
> +
> +my $create_mds = sub {
> +    my ($rados, $id) = @_;
> +
> +    my $priv = [
> +	mon => 'allow profile mds',
> +	osd => 'allow rwx',
> +	mds => 'allow *',
> +    ];
> +
> +    # $id needs to be literally in the config
> +    my $keyring = '/var/lib/ceph/mds/ceph-$id/keyring';
> +
> +    my $cfg = PVE::CephTools::parse_ceph_config();
> +
> +    if (!defined($cfg->{'mds'}->{'keyring'})) {
> +	$cfg->{'mds'}->{'keyring'} = $keyring;
> +	PVE::CephTools::write_ceph_config($cfg);
> +    }
> +
> +    PVE::CephTools::create_ceph_service($rados, $id, 'mds', $priv);
> +};
> +my $destroy_mds = sub {
> +    my ($rados, $mds_id) = @_;
> +
> +    PVE::CephTools::destroy_ceph_service($rados, $mds_id, 'mds');
> +};
> +

if you want to use sub paths here, like 'mds' and 'mds/{id}' you need to let
the resthandler now about them, i.e., adda index GET method with '' as path here,
something like:

__PACKAGE__->register_method ({
    name => 'index',
    path => '',
    method => 'GET',
    description => "Directory index.",
    permissions => { user => 'all' },
    permissions => {
        check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
    },
    parameters => {
        additionalProperties => 0,
        properties => {
            node => get_standard_option('pve-node'),
        },
    },
    returns => {
        type => 'array',
        items => {
            type => "object",
            properties => {},
        },
        links => [ { rel => 'child', href => "{name}" } ],
    },
    code => sub {
        my ($param) = @_;

        my $result = [
            { name => 'mds' },
        ];

        return $result;
    }});

(note, the name is not important, just important that a GET on the fs/ path (so for 
this module the '' path, as it sets paths already relative to .../ceph/fs/ ) with the
respective return schema (links) is defined.

> +__PACKAGE__->register_method ({
> +    name => 'createmds',
> +    path => 'mds',

if you use {id} as template in the DELETE call it'd do it here too, see also
post only once principle. so:

path => 'mds/{id}',

To make this and the DELETE below work in the API you need a GET on the 'mds' path
which returns all IDs. Such a call is desired independent of our API functionality
as the ability to delete a MDS with a certain ID is not really helpful if there's
no way to find out what IDs there are anyway.
There are no references in the CEPH config, AFAICT, so maybe just check which systemnd
ceph-mds@ templates are available, e.g. in
/etc/systemd/system/ceph-mds.target.wants/ceph-mds@<MDS-ID>.service

> +    method => 'POST',
> +    description => "Create Ceph Metadata Server",
> +    proxyto => 'node',
> +    protected => 1,
> +    permissions => {
> +	check => ['perm', '/', [ 'Sys.Modify' ]],
> +    },
> +    parameters => {
> +	additionalProperties => 0,
> +	properties => {
> +	    node => get_standard_option('pve-node'),
> +	    id => {
> +		type => 'string',
> +		optional => 1,
> +		pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
> +		description => "The ID for the mds, when omitted the same as the nodename",
> +	    },
> +	},
> +    },
> +    returns => { type => 'string' },
> +    code => sub {
> +	my ($param) = @_;
> +
> +	PVE::CephTools::check_ceph_installed('ceph_mds');
> +
> +	PVE::CephTools::check_ceph_inited();
> +
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +
> +	my $authuser = $rpcenv->get_user();
> +
> +	my $mdsid = $param->{id} // $param->{node};
> +
> +	my $worker = sub  {
> +	    my $upid = shift;
> +
> +	    my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
> +
> +	    $create_mds->($rados, $mdsid);
> +	};
> +
> +	return $rpcenv->fork_worker('cephcreatemds', "mds.$mdsid", $authuser, $worker);
> +    }});
> +
> +__PACKAGE__->register_method ({
> +    name => 'destroymds',
> +    path => 'mds/{id}',
> +    method => 'DELETE',
> +    description => "Destroy Ceph Metadata Server",
> +    proxyto => 'node',
> +    protected => 1,
> +    permissions => {
> +	check => ['perm', '/', [ 'Sys.Modify' ]],
> +    },
> +    parameters => {
> +	additionalProperties => 0,
> +	properties => {
> +	    node => get_standard_option('pve-node'),
> +	    id => {
> +		description => 'The ID of the mds',
> +		type => 'string',
> +		pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
> +	    },
> +	},
> +    },
> +    returns => { type => 'string' },
> +    code => sub {
> +	my ($param) = @_;
> +
> +	my $rpcenv = PVE::RPCEnvironment::get();
> +
> +	my $authuser = $rpcenv->get_user();
> +
> +	PVE::CephTools::check_ceph_inited();
> +
> +	my $mdsid = $param->{id};
> +
> +	my $worker = sub {
> +	    my $upid = shift;
> +
> +	    my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
> +
> +	    $destroy_mds->($rados, $mdsid);
> +	};
> +
> +	return $rpcenv->fork_worker('cephdestroymds', "mds.$mdsid",  $authuser, $worker);
> +    }});
> +
> diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
> index a5a04949..02ed565e 100755
> --- a/PVE/CLI/pveceph.pm
> +++ b/PVE/CLI/pveceph.pm
> @@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
>  use PVE::JSONSchema qw(get_standard_option);
>  use PVE::CephTools;
>  use PVE::API2::Ceph;
> +use PVE::API2::CephFS;
>  
>  use PVE::CLIHandler;
>  
> @@ -175,6 +176,8 @@ our $cmddef = {
>      destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
>      createmgr => [ 'PVE::API2::Ceph', 'createmgr', [], { node => $nodename }, $upid_exit],
>      destroymgr => [ 'PVE::API2::Ceph', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
> +    createmds => [ 'PVE::API2::CephFS', 'createmds', [], { node => $nodename }, $upid_exit],
> +    destroymds => [ 'PVE::API2::CephFS', 'destroymds', ['id'], { node => $nodename }, $upid_exit],
>      start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, $upid_exit],
>      stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, $upid_exit],
>      install => [ __PACKAGE__, 'install', [] ],
> diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
> index b104b5e8..25160d73 100644
> --- a/PVE/CephTools.pm
> +++ b/PVE/CephTools.pm
> @@ -24,7 +24,8 @@ my $ceph_service = {
>      ceph_bin => "/usr/bin/ceph",
>      ceph_mon => "/usr/bin/ceph-mon",
>      ceph_mgr => "/usr/bin/ceph-mgr",
> -    ceph_osd => "/usr/bin/ceph-osd"
> +    ceph_osd => "/usr/bin/ceph-osd",
> +    ceph_mds => "/usr/bin/ceph-mds"
>  };
>  
>  my $config_hash = {
> @@ -228,4 +229,74 @@ sub systemd_managed {
>      }
>  }
>  
> +sub create_ceph_service {
> +    my ($rados, $id, $service, $priv) = @_;
> +
> +    # eg. ceph fs status fails with numeric only ID.
> +    die "ID: $id, numeric only IDs are not supported\n"
> +	if $id =~ /^\d+$/;
> +
> +    # TODO: add check if service already exists in the cluster with the same ID
> +
> +    my $cluster_name = PVE::CephTools::get_config('ccname');
> +    my $service_dir = "/var/lib/ceph/$service/$cluster_name-$id";
> +    my $service_keyring = "$service_dir/keyring";
> +    my $service_name = "$service.$id";
> +
> +    die "ceph $service directory '$service_dir' already exists\n"
> +	if -d $service_dir;
> +
> +    print "creating $service directory '$service_dir'\n";
> +    eval { File::Path::mkpath($service_dir) };
> +    my $err = $@;
> +    die "creation $service directory '$service_dir' failed\n" if $err;
> +
> +    print "creating keys for '$service_name'\n";
> +    my $output = $rados->mon_command({ prefix => 'auth get-or-create',
> +				       entity => $service_name,
> +				       caps => $priv,
> +				       format => 'plain'});
> +    PVE::Tools::file_set_contents($service_keyring, $output);
> +
> +    print "setting owner for directory\n";
> +    run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
> +
> +    print "enabling service 'ceph-$service\@$id.service'\n";
> +    ceph_service_cmd('enable', $service_name);
> +    print "starting service 'ceph-$service\@$id.service'\n";
> +    ceph_service_cmd('start', $service_name);
> +
> +    return undef;
> +};
> +
> +sub destroy_ceph_service {
> +    my ($rados, $id, $service) = @_;
> +
> +    my $cluster_name = PVE::CephTools::get_config('ccname');
> +    my $service_name = "$service.$id";
> +    my $service_dir = "/var/lib/ceph/$service/$cluster_name-$id";
> +
> +    warn "ceph $service directory '$service_dir' not found\n"
> +	if ! -d $service_dir;
> +
> +    print "disabling service 'ceph-$service\@$id.service'\n";
> +    PVE::CephTools::ceph_service_cmd('disable', $service_name);
> +    print "stopping service 'ceph-$service\@$id.service'\n";
> +    PVE::CephTools::ceph_service_cmd('stop', $service_name);
> +
> +    if (-d $service_dir) {
> +	print "removing ceph-$service directory '$service_dir'\n";
> +	File::Path::remove_tree($service_dir);
> +    }
> +
> +    print "removing ceph user $service_name\n";
> +    $rados->mon_command({
> +	    prefix => 'auth del',
> +	    entity => $service_name,
> +	    format => 'plain'
> +	});
> +
> +    return undef;
> +};
> +
>  1;
>