[pve-devel] [PATCH pve-manager 1/3] pveceph: create mgr with mon, use nodename for id

[Wolfgang Bumiller]

On Mon, Aug 14, 2017 at 09:36:11AM +0200, Alwin Antreich wrote:
> From: Dominik Csapak <d.csapak at proxmox.com>
> 
> we now want to add a ceph-mgr daemon to every node where a ceph-mon
> daemon runs, as per ceph documentation recommendation, because in
> luminous the mgr daemons will not be automatically created/started
> with a monitor anymore
> 
> we also give the createmon an optional id parameter, so that one
> can set a custom id, and make the creation/removal of the manager
> optional but the default
> 
> Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
> (cherry picked from commit f77a087446390840846cd9300ce3dbe7f4bc7fd6)
> Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
> ---
>  PVE/API2/Ceph.pm | 106 +++++++++++++++++++++++++++++++++++++++++++++++--------
>  PVE/CephTools.pm |   2 +-
>  2 files changed, 92 insertions(+), 16 deletions(-)
> 
> diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
> index a63199bc..c36f31ff 100644
> --- a/PVE/API2/Ceph.pm
> +++ b/PVE/API2/Ceph.pm
> @@ -809,11 +809,63 @@ my $find_node_ip = sub {
>      die "unable to find local address within network '$cidr'\n";
>  };
>  
> +my $create_mgr = sub {
> +    my ($rados, $id) = @_;
> +
> +    my $clustername = PVE::CephTools::get_config('ccname');
> +    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
> +    my $mgrkeyring = "$mgrdir/keyring";
> +    my $mgrname = "mgr.$id";
> +
> +    die "ceph manager directory '$mgrdir' already exists\n"
> +	if -d $mgrdir;
> +
> +    print "creating manager directory '$mgrdir'\n";
> +    mkdir $mgrdir;
> +    print "creating keys for '$mgrname'\n";
> +    my $output = $rados->mon_command({ prefix => 'auth get-or-create',
> +				       entity => $mgrname,
> +				       caps => [
> +					   mon => 'allow profile mgr',
> +					   osd => 'allow *',
> +					   mds => 'allow *',
> +				       ],
> +				       format => 'plain'});
> +    PVE::Tools::file_set_contents($mgrkeyring, $output);
> +
> +    print "setting owner for directory\n";
> +    run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
> +
> +    print "enabling service 'ceph-mgr\@$id.service'\n";
> +    PVE::CephTools::ceph_service_cmd('enable', $mgrname);
> +    print "starting service 'ceph-mgr\@$id.service'\n";
> +    PVE::CephTools::ceph_service_cmd('start', $mgrname);
> +};
> +
> +my $destroy_mgr = sub {
> +    my ($mgrid) = @_;
> +
> +    my $clustername = PVE::CephTools::get_config('ccname');
> +    my $mgrname = "mgr.$mgrid";
> +    my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
> +
> +    die "ceph manager directory '$mgrdir' not found\n"
> +	if ! -d $mgrdir;
> +
> +    print "disabling service 'ceph-mgr\@$mgrid.service'\n";
> +    PVE::CephTools::ceph_service_cmd('disable', $mgrname);
> +    print "stopping service 'ceph-mgr\@$mgrid.service'\n";
> +    PVE::CephTools::ceph_service_cmd('stop', $mgrname);
> +
> +    print "removing manager directory '$mgrdir'\n";
> +    File::Path::remove_tree($mgrdir);
> +};
> +
>  __PACKAGE__->register_method ({
>      name => 'createmon',
>      path => 'mon',
>      method => 'POST',
> -    description => "Create Ceph Monitor",
> +    description => "Create Ceph Monitor and Manager",
>      proxyto => 'node',
>      protected => 1,
>      permissions => {
> @@ -823,6 +875,18 @@ __PACKAGE__->register_method ({
>      	additionalProperties => 0,
>  	properties => {
>  	    node => get_standard_option('pve-node'),
> +	    id => {
> +		type => 'string',
> +		optional => 1,
> +		pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
> +		description => "The ID for the monitor, when omitted the same as the nodename",
> +	    },
> +	    'exclude-manager' => {
> +		type => 'boolean',
> +		optional => 1,
> +		default => 0,
> +		description => "When set, only a monitor will be created.",
> +	    },
>  	},
>      },
>      returns => { type => 'string' },
> @@ -856,14 +920,7 @@ __PACKAGE__->register_method ({
>  	    }
>  	}
>  
> -	my $monid;
> -	for (my $i = 0; $i < 7; $i++) {
> -	    if (!$cfg->{"mon.$i"}) {
> -		$monid = $i;
> -		last;
> -	    }
> -	}
> -	die "unable to find usable monitor id\n" if !defined($monid);
> +	my $monid = $param->{id} // $param->{node};
>  
>  	my $monsection = "mon.$monid"; 
>  	my $ip;
> @@ -909,14 +966,15 @@ __PACKAGE__->register_method ({
>  	    -d $mondir && die "monitor filesystem '$mondir' already exist\n";
>   
>  	    my $monmap = "/tmp/monmap";
> -	
> +
> +	    my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
> +
>  	    eval {
>  		mkdir $mondir;
>  
>  		run_command("chown ceph:ceph $mondir") if $systemd_managed;
>  
>  		if ($moncount > 0) {
> -		    my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
>  		    my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
>  		    PVE::Tools::file_set_contents($monmap, $mapdata);
>  		} else {
> @@ -947,6 +1005,11 @@ __PACKAGE__->register_method ({
>  		eval { PVE::Tools::run_command(['/bin/systemctl', 'enable', "ceph-mon\@${monid}.service"]); };
>  		warn "Enable ceph-mon\@${monid}.service manually"if $@;
>  	    }
> +
> +	    # create manager
> +	    if (!$param->{'exclude-manager'}) {

Patch 3 should probably add a call to check_ceph_mgr_installed() here as
well with the $noerr parameter set, so that the default behavior still
works when not having upgraded yet.

> +		$create_mgr->($rados, $monid);
> +	    }
>  	};
>  
>  	return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
> @@ -956,7 +1019,7 @@ __PACKAGE__->register_method ({
>      name => 'destroymon',
>      path => 'mon/{monid}',
>      method => 'DELETE',
> -    description => "Destroy Ceph monitor.",
> +    description => "Destroy Ceph Monitor and Manager.",
>      proxyto => 'node',
>      protected => 1,
>      permissions => {