[pve-devel] [PATCH v3 pve-manager 12/13] fix #4759: ceph: configure ceph-crash.service and its key
Max Carrara
m.carrara at proxmox.com
Fri Feb 16 15:56:14 CET 2024
Due to Ceph dropping privileges when running the 'ceph-crash' daemon
[0], it is necessary to allow the daemon to authenticate with its
cluster in a safe manner.
In order to avoid exposing sensitive keyrings or somehow escalating
its privileges again, 'ceph-crash' is therefore provided with its own
keyring in the '/etc/pve/ceph' directory. This directory, due to being
on 'pmxcfs', may be read by members of the 'www-data' group, which
'ceph-crash' is made part of [1].
Expected Configuration
----------------------
1. A keyring file named '/etc/pve/ceph/ceph.client.crash.keyring'
exists
2. A section named 'client.crash' exists in '/etc/pve/ceph.conf'
3. The 'client.crash' section has a key named 'keyring' which
references the keyring file as '/etc/pve/ceph/$cluster.$name.keyring'
New Clusters
------------
The keyring file is created and the conf file is updated after the first
monitor has been created (when calling `pveceph mon create`).
Existing Clusters
-----------------
A new helper script creates and configures the 'client.crash' keyring in
`postinst`, if:
* Ceph is installed
* Ceph is initialized ('/etc/pve/ceph.conf' and '/etc/pve/ceph' exist)
* Connection to RADOS is successful
If the above conditions are met, the helper script ensures that the
existing configuration matches the expected configuration mentioned
above.
The configuration is not changed if it is already as expected.
The helper script may be called again manually if the `postinst` hook
fails. It is installed to '/usr/share/pve-manager/helpers/pve-init-ceph-crash'.
Existing `client.crash` Key
---------------------------
If a key named 'client.crash' already exists within the cluster, it is
reused and not regenerated.
[0]: https://github.com/ceph/ceph/pull/48713
[1]: https://git.proxmox.com/?p=ceph.git;a=commitdiff;h=f72c698a55905d93e9a0b7b95674616547deba8a
Signed-off-by: Max Carrara <m.carrara at proxmox.com>
---
Changes v1 --> v2:
* fix 'keyring' key being appended to 'client.crash' section even
if it already exists and configured correctly
Changes v2 --> v3:
* avoid needless second 'pmxcfs' lock on 'ceph.conf' when configuring
'client.crash' keyring after first monitor was created
* change name of helper sub that creates the ceph.crash keyring
* the helper sub now implicitly expects the /etc/pve/ceph directory to
exist (as the previous commit enforces its existence)
* remove BASH part of postinst hook related to `ceph-crash`
* add Perl helper script and call that instead in postinst hook
* reword commit message
PVE/API2/Ceph/MON.pm | 8 ++++
PVE/Ceph/Tools.pm | 35 ++++++++++++++
bin/Makefile | 1 +
bin/pve-init-ceph-crash | 104 ++++++++++++++++++++++++++++++++++++++++
debian/postinst | 4 ++
5 files changed, 152 insertions(+)
create mode 100755 bin/pve-init-ceph-crash
diff --git a/PVE/API2/Ceph/MON.pm b/PVE/API2/Ceph/MON.pm
index 1e959ef3..8b7ce376 100644
--- a/PVE/API2/Ceph/MON.pm
+++ b/PVE/API2/Ceph/MON.pm
@@ -450,6 +450,14 @@ __PACKAGE__->register_method ({
)
};
warn "$@" if $@;
+
+ print "Configuring keyring for ceph-crash.service\n";
+ eval {
+ PVE::Ceph::Tools::create_or_update_crash_keyring_file();
+ $cfg->{'client.crash'}->{keyring} = '/etc/pve/ceph/$cluster.$name.keyring';
+ cfs_write_file('ceph.conf', $cfg);
+ };
+ warn "Unable to configure keyring for ceph-crash.service: $@" if $@;
}
eval { PVE::Ceph::Services::ceph_service_cmd('enable', $monsection) };
diff --git a/PVE/Ceph/Tools.pm b/PVE/Ceph/Tools.pm
index 735bb116..9b97171e 100644
--- a/PVE/Ceph/Tools.pm
+++ b/PVE/Ceph/Tools.pm
@@ -20,6 +20,7 @@ my $pve_ceph_cfgpath = "/etc/pve/$ccname.conf";
my $ceph_cfgpath = "$ceph_cfgdir/$ccname.conf";
my $pve_ceph_cfgdir = "/etc/pve/ceph";
+my $pve_ceph_crash_key_path = "$pve_ceph_cfgdir/$ccname.client.crash.keyring";
my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
my $ckeyring_path = "/etc/ceph/ceph.client.admin.keyring";
@@ -45,6 +46,7 @@ my $config_values = {
my $config_files = {
pve_ceph_cfgpath => $pve_ceph_cfgpath,
+ pve_ceph_crash_key_path => $pve_ceph_crash_key_path,
pve_mon_key_path => $pve_mon_key_path,
pve_ckeyring_path => $pve_ckeyring_path,
ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
@@ -420,6 +422,39 @@ sub get_or_create_admin_keyring {
return $pve_ckeyring_path;
}
+# is also used in `pve-init-ceph-crash` helper
+sub create_or_update_crash_keyring_file {
+ my ($rados) = @_;
+
+ if (!defined($rados)) {
+ $rados = PVE::RADOS->new();
+ }
+
+ my $output = $rados->mon_command({
+ prefix => 'auth get-or-create',
+ entity => 'client.crash',
+ caps => [
+ mon => 'profile crash',
+ mgr => 'profile crash',
+ ],
+ format => 'plain',
+ });
+
+ if (-f $pve_ceph_crash_key_path) {
+ my $contents = PVE::Tools::file_get_contents($pve_ceph_crash_key_path);
+
+ if ($contents ne $output) {
+ PVE::Tools::file_set_contents($pve_ceph_crash_key_path, $output);
+ return 1;
+ }
+ } else {
+ PVE::Tools::file_set_contents($pve_ceph_crash_key_path, $output);
+ return 1;
+ }
+
+ return 0;
+}
+
# get ceph-volume managed osds
sub ceph_volume_list {
my $result = {};
diff --git a/bin/Makefile b/bin/Makefile
index 06d8148e..b221e4b1 100644
--- a/bin/Makefile
+++ b/bin/Makefile
@@ -83,6 +83,7 @@ install: $(SCRIPTS) $(CLI_MANS) $(SERVICE_MANS) $(BASH_COMPLETIONS) $(ZSH_COMPLE
install -m 0755 $(SCRIPTS) $(BINDIR)
install -d $(USRSHARE)/helpers
install -m 0755 pve-startall-delay $(USRSHARE)/helpers
+ install -m 0755 pve-init-ceph-crash $(USRSHARE)/helpers
install -d $(MAN1DIR)
install -m 0644 $(CLI_MANS) $(MAN1DIR)
install -d $(MAN8DIR)
diff --git a/bin/pve-init-ceph-crash b/bin/pve-init-ceph-crash
new file mode 100755
index 00000000..f7c2f6e4
--- /dev/null
+++ b/bin/pve-init-ceph-crash
@@ -0,0 +1,104 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use List::Util qw(first);
+
+use PVE::Ceph::Tools;
+use PVE::Cluster;
+use PVE::RADOS;
+use PVE::RPCEnvironment;
+
+
+sub main {
+ my $rpcenv = PVE::RPCEnvironment->init('cli');
+
+ $rpcenv->init_request();
+ $rpcenv->set_language($ENV{LANG});
+ $rpcenv->set_user('root at pam');
+
+ die "Not running as root. Exiting.\n" if $> != 0;
+
+ print("Checking whether keyring for 'ceph-crash.service' needs to be configured...\n");
+
+ if (!PVE::Ceph::Tools::check_ceph_installed('ceph_bin', 1)) {
+ print("Ceph is not installed. No action required.\n");
+ exit 0;
+ }
+
+ eval {
+ PVE::Ceph::Tools::check_ceph_inited();
+ };
+ if ($@) {
+ print("Ceph is not initialized. No action required.\n");
+ exit 0;
+ }
+
+ my $rados = eval { PVE::RADOS->new() };
+ $@ = undef; # warnings are displayed regardless
+
+ my $ceph_cfg_file = 'ceph.conf';
+ my $ceph_crash_key_path = PVE::Ceph::Tools::get_config('pve_ceph_crash_key_path');
+
+ my $keyring_value = '/etc/pve/ceph/$cluster.$name.keyring';
+
+ my $changed = 0;
+
+ PVE::Cluster::cfs_lock_file($ceph_cfg_file, undef, sub {
+ my $cfg = PVE::Cluster::cfs_read_file($ceph_cfg_file);
+
+ if (!defined($rados)) {
+ my $has_mon_config = defined(first { m/mon\..*/ } keys $cfg->%*);
+ if ($has_mon_config) {
+ die "Connection to RADOS failed even though a monitor is configured.\n" .
+ "Please verify whether your configuration is correct.\n"
+ }
+
+ print(
+ "Connection to RADOS failed and no monitor is configured. ".
+ "Assume that things are fine. No action required.\n"
+ );
+ return undef;
+ }
+
+ my $write_to_conf = sub {
+ print("Updating Ceph configuration...\n");
+ $cfg->{'client.crash'}->{keyring} = $keyring_value;
+ PVE::Cluster::cfs_write_file($ceph_cfg_file, $cfg);
+ $changed = 1;
+ };
+
+ $changed = PVE::Ceph::Tools::create_or_update_crash_keyring_file($rados);
+
+ if (!exists($cfg->{'client.crash'})) {
+ $write_to_conf->();
+ return undef;
+ }
+
+ if (!exists($cfg->{'client.crash'}->{keyring})) {
+ $write_to_conf->();
+ return undef;
+ }
+
+ my $current_keyring_value = $cfg->{'client.crash'}->{keyring};
+ if ($current_keyring_value ne $keyring_value) {
+ $write_to_conf->();
+ return undef;
+ }
+
+ });
+ if ($@) {
+ warn("Failed to configure keyring for 'ceph-crash.service'. Error:\n");
+ warn("$@");
+ exit 1;
+ }
+
+ if ($changed) {
+ print("Successfully updated configuration for 'ceph-crash.service'.\n");
+ }
+}
+
+main();
+
+exit 0;
diff --git a/debian/postinst b/debian/postinst
index 61b50f97..c989dac1 100755
--- a/debian/postinst
+++ b/debian/postinst
@@ -86,6 +86,10 @@ update_ceph_conf() {
if test ! -d "${CEPH_CONF_DIR}"; then
mkdir -p "${CEPH_CONF_DIR}"
fi
+
+ # Don't fail in case user has "exotic" configuration where RADOS
+ # isn't available on all nodes for some reason
+ /usr/share/pve-manager/helpers/pve-init-ceph-crash || true
}
migrate_apt_auth_conf() {
--
2.39.2
More information about the pve-devel
mailing list