[pve-devel] [PATCH manager v3 1/3] pveceph: create mgr with mon, use nodename for id
Dominik Csapak
d.csapak at proxmox.com
Wed Jul 26 11:11:51 CEST 2017
we now want to add a ceph-mgr daemon to every node where a ceph-mon
daemon runs, as per ceph documentation recommendation, because in
luminous the mgr daemons will not be automatically created/started
with a monitor anymore
we also give the createmon an optional id parameter, so that one
can set a custom id, and make the creation/removal of the manager
optional but the default
Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
---
changes from v2:
* changed $mgrname to $mgrdir in the 'not found' die statement
PVE/API2/Ceph.pm | 106 +++++++++++++++++++++++++++++++++++++++++++++++--------
PVE/CephTools.pm | 2 +-
2 files changed, 92 insertions(+), 16 deletions(-)
diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
index 105ee37a..9bcac311 100644
--- a/PVE/API2/Ceph.pm
+++ b/PVE/API2/Ceph.pm
@@ -843,11 +843,63 @@ my $find_node_ip = sub {
die "unable to find local address within network '$cidr'\n";
};
+my $create_mgr = sub {
+ my ($rados, $id) = @_;
+
+ my $clustername = PVE::CephTools::get_config('ccname');
+ my $mgrdir = "/var/lib/ceph/mgr/$clustername-$id";
+ my $mgrkeyring = "$mgrdir/keyring";
+ my $mgrname = "mgr.$id";
+
+ die "ceph manager directory '$mgrdir' already exists\n"
+ if -d $mgrdir;
+
+ print "creating manager directory '$mgrdir'\n";
+ mkdir $mgrdir;
+ print "creating keys for '$mgrname'\n";
+ my $output = $rados->mon_command({ prefix => 'auth get-or-create',
+ entity => $mgrname,
+ caps => [
+ mon => 'allow profile mgr',
+ osd => 'allow *',
+ mds => 'allow *',
+ ],
+ format => 'plain'});
+ PVE::Tools::file_set_contents($mgrkeyring, $output);
+
+ print "setting owner for directory\n";
+ run_command(["chown", 'ceph:ceph', '-R', $mgrdir]);
+
+ print "enabling service 'ceph-mgr\@$id.service'\n";
+ PVE::CephTools::ceph_service_cmd('enable', $mgrname);
+ print "starting service 'ceph-mgr\@$id.service'\n";
+ PVE::CephTools::ceph_service_cmd('start', $mgrname);
+};
+
+my $destroy_mgr = sub {
+ my ($mgrid) = @_;
+
+ my $clustername = PVE::CephTools::get_config('ccname');
+ my $mgrname = "mgr.$mgrid";
+ my $mgrdir = "/var/lib/ceph/mgr/$clustername-$mgrid";
+
+ die "ceph manager directory '$mgrdir' not found\n"
+ if ! -d $mgrdir;
+
+ print "disabling service 'ceph-mgr\@$mgrid.service'\n";
+ PVE::CephTools::ceph_service_cmd('disable', $mgrname);
+ print "stopping service 'ceph-mgr\@$mgrid.service'\n";
+ PVE::CephTools::ceph_service_cmd('stop', $mgrname);
+
+ print "removing manager directory '$mgrdir'\n";
+ File::Path::remove_tree($mgrdir);
+};
+
__PACKAGE__->register_method ({
name => 'createmon',
path => 'mon',
method => 'POST',
- description => "Create Ceph Monitor",
+ description => "Create Ceph Monitor and Manager",
proxyto => 'node',
protected => 1,
permissions => {
@@ -857,6 +909,18 @@ __PACKAGE__->register_method ({
additionalProperties => 0,
properties => {
node => get_standard_option('pve-node'),
+ id => {
+ type => 'string',
+ optional => 1,
+ pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
+ description => "The ID for the monitor, when omitted the same as the nodename",
+ },
+ 'exclude-manager' => {
+ type => 'boolean',
+ optional => 1,
+ default => 0,
+ description => "When set, only a monitor will be created.",
+ },
},
},
returns => { type => 'string' },
@@ -890,14 +954,7 @@ __PACKAGE__->register_method ({
}
}
- my $monid;
- for (my $i = 0; $i < 7; $i++) {
- if (!$cfg->{"mon.$i"}) {
- $monid = $i;
- last;
- }
- }
- die "unable to find usable monitor id\n" if !defined($monid);
+ my $monid = $param->{id} // $param->{node};
my $monsection = "mon.$monid";
my $ip;
@@ -944,14 +1001,15 @@ __PACKAGE__->register_method ({
-d $mondir && die "monitor filesystem '$mondir' already exist\n";
my $monmap = "/tmp/monmap";
-
+
+ my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
+
eval {
mkdir $mondir;
run_command("chown ceph:ceph $mondir") if $systemd_managed;
if ($moncount > 0) {
- my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
my $mapdata = $rados->mon_command({ prefix => 'mon getmap', format => 'plain' });
PVE::Tools::file_set_contents($monmap, $mapdata);
} else {
@@ -990,6 +1048,11 @@ __PACKAGE__->register_method ({
}
waitpid($create_keys_pid, 0);
}
+
+ # create manager
+ if (!$param->{'exclude-manager'}) {
+ $create_mgr->($rados, $monid);
+ }
};
return $rpcenv->fork_worker('cephcreatemon', $monsection, $authuser, $worker);
@@ -999,7 +1062,7 @@ __PACKAGE__->register_method ({
name => 'destroymon',
path => 'mon/{monid}',
method => 'DELETE',
- description => "Destroy Ceph monitor.",
+ description => "Destroy Ceph Monitor and Manager.",
proxyto => 'node',
protected => 1,
permissions => {
@@ -1011,8 +1074,15 @@ __PACKAGE__->register_method ({
node => get_standard_option('pve-node'),
monid => {
description => 'Monitor ID',
- type => 'integer',
+ type => 'string',
+ pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
},
+ 'exclude-manager' => {
+ type => 'boolean',
+ default => 0,
+ optional => 1,
+ description => "When set, removes only the monitor, not the manager"
+ }
},
},
returns => { type => 'string' },
@@ -1058,6 +1128,12 @@ __PACKAGE__->register_method ({
delete $cfg->{$monsection};
PVE::CephTools::write_ceph_config($cfg);
File::Path::remove_tree($mondir);
+
+ # remove manager
+ if (!$param->{'exclude-manager'}) {
+ eval { $destroy_mgr->($monid); };
+ warn $@ if $@;
+ }
};
return $rpcenv->fork_worker('cephdestroymon', $monsection, $authuser, $worker);
@@ -1081,7 +1157,7 @@ __PACKAGE__->register_method ({
description => 'Ceph service name.',
type => 'string',
optional => 1,
- pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
+ pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9]{1,32}',
},
},
},
@@ -1131,7 +1207,7 @@ __PACKAGE__->register_method ({
description => 'Ceph service name.',
type => 'string',
optional => 1,
- pattern => '(mon|mds|osd)\.[A-Za-z0-9]{1,32}',
+ pattern => '(mon|mds|osd|mgr)\.[A-Za-z0-9]{1,32}',
},
},
},
diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
index 0d26ea3e..0c0d7c18 100644
--- a/PVE/CephTools.pm
+++ b/PVE/CephTools.pm
@@ -178,7 +178,7 @@ sub ceph_service_cmd {
if (systemd_managed()) {
- if ($service && $service =~ m/^(mon|osd|mds|radosgw)(\.([A-Za-z0-9]{1,32}))?$/) {
+ if ($service && $service =~ m/^(mon|osd|mds|mgr|radosgw)(\.([A-Za-z0-9\-]{1,32}))?$/) {
$service = defined($3) ? "ceph-$1\@$3" : "ceph-$1.target";
} else {
$service = "ceph.target";
--
2.11.0
More information about the pve-devel
mailing list