[pve-devel] [PATCH v3 storage 1/2] Fix #1542: show storage utilization per pool

Alwin Antreich a.antreich at proxmox.com
Wed Apr 11 16:36:41 CEST 2018


 - get the percent_used value for a ceph pool and
   calculate it where ceph doesn't supply it (pre kraken)
 - use librados2-perl for pool status
 - add librados2-perl as build-depends and depends in debian/control

Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
 PVE/CLI/pvesm.pm         |  1 +
 PVE/Storage.pm           |  4 +-
 PVE/Storage/RBDPlugin.pm | 96 ++++++++++++++++++++++++++++++++++++++----------
 debian/control           |  2 +
 4 files changed, 81 insertions(+), 22 deletions(-)

diff --git a/PVE/CLI/pvesm.pm b/PVE/CLI/pvesm.pm
index 5774364..98cd9e9 100755
--- a/PVE/CLI/pvesm.pm
+++ b/PVE/CLI/pvesm.pm
@@ -149,6 +149,7 @@ my $print_status = sub {
 	my $active = $res->{active} ? 'active' : 'inactive';
 	my ($per, $per_fmt) = (0, '% 7.2f%%');
 	$per = ($res->{used}*100)/$res->{total} if $res->{total} > 0;
+	$per = $res->{percent_used} if defined($res->{percent_used});
 
 	if (!$res->{enabled}) {
 	    $per = 'N/A';
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 4140a99..0d9d7cf 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1065,14 +1065,14 @@ sub storage_info {
 	    next;
 	}
 
-	my ($total, $avail, $used, $active);
-	eval { ($total, $avail, $used, $active) = $plugin->status($storeid, $scfg, $cache); };
+	my ($total, $avail, $used, $active, $percent_used) = eval { $plugin->status($storeid, $scfg, $cache); };
 	warn $@ if $@;
 	next if !$active;
 	$info->{$storeid}->{total} = int($total);
 	$info->{$storeid}->{avail} = int($avail);
 	$info->{$storeid}->{used} = int($used);
 	$info->{$storeid}->{active} = $active;
+	$info->{$storeid}->{percent_used} = $percent_used if (defined($percent_used));
     }
 
     return $info;
diff --git a/PVE/Storage/RBDPlugin.pm b/PVE/Storage/RBDPlugin.pm
index fd5a2ef..1b09b58 100644
--- a/PVE/Storage/RBDPlugin.pm
+++ b/PVE/Storage/RBDPlugin.pm
@@ -7,6 +7,7 @@ use Net::IP;
 use PVE::Tools qw(run_command trim);
 use PVE::Storage::Plugin;
 use PVE::JSONSchema qw(get_standard_option);
+use PVE::RADOS;
 
 use base qw(PVE::Storage::Plugin);
 
@@ -90,6 +91,50 @@ my $rados_cmd = sub {
     return $build_cmd->('/usr/bin/rados', $scfg, $storeid, $op, @options);
 };
 
+my $ceph_connect_option = sub {
+    my ($scfg, $storeid, %options) = @_;
+
+    my $cmd_option = {};
+    my $ceph_storeid_conf = "/etc/pve/priv/ceph/${storeid}.conf";
+    my $keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
+    my $pveceph_managed = !defined($scfg->{monhost});
+
+    $cmd_option->{ceph_conf} = $pveceph_config if (-e $pveceph_config);
+
+    if (-e $ceph_storeid_conf) {
+	if ($pveceph_managed) {
+	    warn "ignoring custom ceph config for storage '$storeid', 'monhost' is not set (assuming pveceph managed cluster)!\n";
+	} else {
+	    $cmd_option->{ceph_conf} = $ceph_storeid_conf;
+	}
+    }
+
+    $cmd_option->{keyring} = $keyring if (-e $keyring);
+    $cmd_option->{auth_supported} = (defined $cmd_option->{keyring}) ? 'cephx' : 'none';
+    $cmd_option->{userid} =  $scfg->{username} ? $scfg->{username} : 'admin';
+    $cmd_option->{mon_host} = $hostlist->($scfg->{monhost}, ',') if (defined($scfg->{monhost}));
+
+    if (%options) {
+	foreach my $k (keys %options) {
+	    $cmd_option->{$k} = $options{$k};
+	}
+    }
+
+
+    return $cmd_option;
+
+};
+
+my $librados_connect = sub {
+    my ($scfg, $storeid, $options) = @_;
+
+    my $librados_config = $ceph_connect_option->($scfg, $storeid);
+
+    my $rados = PVE::RADOS->new(%$librados_config);
+
+    return $rados;
+};
+
 # needed for volumes created using ceph jewel (or higher)
 my $krbd_feature_disable = sub {
     my ($scfg, $storeid, $name) = @_;
@@ -539,31 +584,42 @@ sub list_images {
 sub status {
     my ($class, $storeid, $scfg, $cache) = @_;
 
-    my $cmd = &$rados_cmd($scfg, $storeid, 'df');
-
-    my $stats = {};
 
-    my $parser = sub {
-	my $line = shift;
-	if ($line =~ m/^\s*total(?:\s|_)(\S+)\s+(\d+)(k|M|G|T)?/) {
-	    $stats->{$1} = $2;
-	    # luminous has units here..
-	    if ($3) {
-		$stats->{$1} *= $rbd_unittobytes->{$3}/1024;
-	    }
-	}
-    };
+    my $rados = &$librados_connect($scfg, $storeid);
+    my $df = $rados->mon_command({ prefix => 'df', format => 'json' });
 
-    eval {
-	run_rbd_command($cmd, errmsg => "rados error", errfunc => sub {}, outfunc => $parser);
-    };
+    my ($d) = grep { $_->{name} eq $scfg->{pool} } @{$df->{pools}};
 
-    my $total = $stats->{space} ? $stats->{space}*1024 : 0;
-    my $free = $stats->{avail} ? $stats->{avail}*1024 : 0;
-    my $used = $stats->{used} ? $stats->{used}*1024: 0;
+    # max_avail -> max available space for data w/o replication in the pool
+    # bytes_used -> data w/o replication in the pool
+    # precent_used -> ceph's %-used of the pool
+    my $free = $d->{stats}->{max_avail};
+    my $used = $d->{stats}->{bytes_used};
+    my $percent_used = $d->{stats}->{percent_used};
+    my $total = $used + $free;
     my $active = 1;
 
-    return ($total, $free, $used, $active);
+    if (!defined($percent_used)) {
+	my $pg_dump = $rados->mon_command({ prefix => 'pg dump', format => 'json' });
+	my ($pool) = grep { $_->{poolid} eq $d->{id} } @{$pg_dump->{pool_stats}};
+
+	# num_object_copies -> holds the count of all object w/ replication of
+	# the pool
+	# num_objects_degraded -> holds all degraded object w/ replication of
+	# the pool
+	my $pg_num_obj_copies = $pool->{stat_sum}->{num_object_copies};
+	my $pg_num_obj_degraded = $pool->{stat_sum}->{num_objects_degraded};
+
+	# percent_used -> the raw space occupied by bytes_used, reduced by a
+	# %-factor from the number of degraded objects compared to the total
+	# number of objects in the pool, the new bytes_used is devided by the
+	# new max_avail to get the current %-used
+	my $percent_used_c = $used * (($pg_num_obj_copies - $pg_num_obj_degraded) / $pg_num_obj_copies);
+	$percent_used_c = $percent_used_c / ($percent_used_c + $free);
+	$percent_used = ($percent_used_c * 100);
+    }
+
+	return ($total, $free, $used, $active, $percent_used);
 }
 
 sub activate_storage {
diff --git a/debian/control b/debian/control
index 3f39364..2cf585a 100644
--- a/debian/control
+++ b/debian/control
@@ -5,6 +5,7 @@ Maintainer: Proxmox Support Team <support at proxmox.com>
 Build-Depends: debhelper (>= 7.0.50~),
                libpve-common-perl (>= 5.0-28),
                libtest-mockmodule-perl,
+               librados2-perl,
                lintian,
                perl (>= 5.10.0-19),
                pve-doc-generator,
@@ -18,6 +19,7 @@ Depends: cstream,
          libfile-chdir-perl,
          libnet-dbus-perl,
          libpve-common-perl (>= 5.0-28),
+         librados2-perl,
          lvm2,
          nfs-common,
          perl (>= 5.6.0-16),
-- 
2.11.0





More information about the pve-devel mailing list