[pve-devel] [PATCH storage 2/2] Diskmanage: detect osds/journals/etc. created with ceph-volume

Dominik Csapak d.csapak at proxmox.com
Wed May 29 15:48:06 CEST 2019


ceph-volume creates osds/journal/etc. on LVM instead of partitions,
so to detect them, we have to parse the lv_tags of the LVs and
match them with the underlying device

also add tests for this detection

Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
---
i first wanted to use the methods in LVMPlugin, but the way the result
is structured is really different from what we need here, so i simply
added it inline.

in pve-manager we probably want to use the output of ceph-volume lvm list
(we can't here to avoid the dependency on ceph-volume) so
we probably won't reuse this code. if we do want to reuse it after all,
we can still refactor it later

 PVE/Diskmanage.pm                             | 45 +++++++++++++++++++
 test/disk_tests/usages/disklist               |  4 ++
 test/disk_tests/usages/disklist_expected.json | 64 +++++++++++++++++++++++++++
 test/disk_tests/usages/lvs                    |  4 ++
 test/disk_tests/usages/pvs                    |  6 ++-
 test/disk_tests/usages/sdg/device/vendor      |  1 +
 test/disk_tests/usages/sdg/queue/rotational   |  1 +
 test/disk_tests/usages/sdg/size               |  1 +
 test/disk_tests/usages/sdg_udevadm            | 12 +++++
 test/disk_tests/usages/sdh/device/vendor      |  1 +
 test/disk_tests/usages/sdh/queue/rotational   |  1 +
 test/disk_tests/usages/sdh/size               |  1 +
 test/disk_tests/usages/sdh_udevadm            | 12 +++++
 test/disk_tests/usages/sdi/device/vendor      |  1 +
 test/disk_tests/usages/sdi/queue/rotational   |  1 +
 test/disk_tests/usages/sdi/size               |  1 +
 test/disk_tests/usages/sdi_udevadm            | 12 +++++
 test/disk_tests/usages/sdj/device/vendor      |  1 +
 test/disk_tests/usages/sdj/queue/rotational   |  1 +
 test/disk_tests/usages/sdj/size               |  1 +
 test/disk_tests/usages/sdj_udevadm            | 12 +++++
 test/disklist_test.pm                         |  2 +
 22 files changed, 184 insertions(+), 1 deletion(-)
 create mode 100644 test/disk_tests/usages/lvs
 create mode 100644 test/disk_tests/usages/sdg/device/vendor
 create mode 100644 test/disk_tests/usages/sdg/queue/rotational
 create mode 100644 test/disk_tests/usages/sdg/size
 create mode 100644 test/disk_tests/usages/sdg_udevadm
 create mode 100644 test/disk_tests/usages/sdh/device/vendor
 create mode 100644 test/disk_tests/usages/sdh/queue/rotational
 create mode 100644 test/disk_tests/usages/sdh/size
 create mode 100644 test/disk_tests/usages/sdh_udevadm
 create mode 100644 test/disk_tests/usages/sdi/device/vendor
 create mode 100644 test/disk_tests/usages/sdi/queue/rotational
 create mode 100644 test/disk_tests/usages/sdi/size
 create mode 100644 test/disk_tests/usages/sdi_udevadm
 create mode 100644 test/disk_tests/usages/sdj/device/vendor
 create mode 100644 test/disk_tests/usages/sdj/queue/rotational
 create mode 100644 test/disk_tests/usages/sdj/size
 create mode 100644 test/disk_tests/usages/sdj_udevadm

diff --git a/PVE/Diskmanage.pm b/PVE/Diskmanage.pm
index de3e60e..72c1432 100644
--- a/PVE/Diskmanage.pm
+++ b/PVE/Diskmanage.pm
@@ -13,6 +13,7 @@ my $SMARTCTL = "/usr/sbin/smartctl";
 my $ZPOOL = "/sbin/zpool";
 my $SGDISK = "/sbin/sgdisk";
 my $PVS = "/sbin/pvs";
+my $LVS = "/sbin/lvs";
 my $UDEVADM = "/bin/udevadm";
 
 sub verify_blockdev_path {
@@ -235,6 +236,39 @@ sub get_ceph_journals {
     return $journalhash;
 }
 
+# reads the lv_tags and matches them with the devices
+sub get_ceph_volume_infos {
+    my $result = {};
+
+    my $cmd = [$LVS, '-S', 'lv_name=~^osd-','-o','devices,lv_name,lv_tags',
+	       '--noheadings', '--readonly', '--separator', ';'];
+
+    run_command($cmd, outfunc => sub {
+	my $line = shift;
+	$line =~ s/(?:^\s+)|(?:\s+$)//g; # trim
+	my $fields = [split(';', $line)];
+
+	# lvs syntax is /dev/sdX(Y) where Y is the start (which we do not need)
+	my ($dev) = $fields->[0] =~ m|^(/dev/[a-z]+)|;
+	if ($fields->[1] =~ m|^osd-([^-]+)-|) {
+	    my $type = $1;
+	    # we use autovivification here to not concern us with
+	    # creation of empty hashes
+	    if (($type eq 'block' || $type eq 'data') &&
+		$fields->[2] =~ m/ceph.osd_id=([^,])/)
+	    {
+		$result->{$dev}->{osdid} = $1;
+		$result->{$dev}->{bluestore} = ($type eq 'block');
+	    } else {
+		# if $foo is undef $foo++ results in '1' (and is well defined)
+		$result->{$dev}->{$type}++;
+	    }
+	}
+    });
+
+    return $result;
+}
+
 sub get_udev_info {
     my ($dev) = @_;
 
@@ -402,6 +436,7 @@ sub get_disks {
     };
 
     my $journalhash = get_ceph_journals();
+    my $ceph_volume_infos = get_ceph_volume_infos();
 
     my $zfslist = get_zfs_devices();
 
@@ -549,6 +584,16 @@ sub get_disks {
 	    }
 	});
 
+	if ($ceph_volume_infos->{$devpath}) {
+	    $journal_count += $ceph_volume_infos->{$devpath}->{journal} // 0;
+	    $db_count += $ceph_volume_infos->{$devpath}->{db} // 0;
+	    $wal_count += $ceph_volume_infos->{$devpath}->{wal} // 0;
+	    if ($ceph_volume_infos->{$devpath}->{osdid}) {
+		$osdid = $ceph_volume_infos->{$devpath}->{osdid};
+		$bluestore = 1 if $ceph_volume_infos->{$devpath}->{bluestore};
+	    }
+	}
+
 	$used = 'mounted' if $found_mountpoints && !$used;
 	$used = 'LVM' if $found_lvm && !$used;
 	$used = 'ZFS' if $found_zfs && !$used;
diff --git a/test/disk_tests/usages/disklist b/test/disk_tests/usages/disklist
index b5daaf1..9092ce0 100644
--- a/test/disk_tests/usages/disklist
+++ b/test/disk_tests/usages/disklist
@@ -4,3 +4,7 @@ sdc
 sdd
 sde
 sdf
+sdg
+sdh
+sdi
+sdj
diff --git a/test/disk_tests/usages/disklist_expected.json b/test/disk_tests/usages/disklist_expected.json
index 3205bbf..4f9f5cc 100644
--- a/test/disk_tests/usages/disklist_expected.json
+++ b/test/disk_tests/usages/disklist_expected.json
@@ -88,5 +88,69 @@
 	"vendor" : "ATA",
 	"wwn" : "0x0000000000000000",
 	"devpath" : "/dev/sdd"
+    },
+    "sdg" : {
+	"serial" : "SERIAL1",
+	"vendor" : "ATA",
+	"wwn" : "0x0000000000000000",
+	"devpath" : "/dev/sdg",
+	"model" : "MODEL1",
+	"used" : "LVM",
+	"wearout" : "N/A",
+	"health" : "UNKNOWN",
+	"gpt" : 1,
+	"size" : 1536000,
+	"rpm" : 0,
+	"type" : "hdd",
+	"bluestore": 1,
+	"osdid" : 1
+    },
+    "sdh" : {
+	"serial" : "SERIAL1",
+	"vendor" : "ATA",
+	"wwn" : "0x0000000000000000",
+	"devpath" : "/dev/sdh",
+	"model" : "MODEL1",
+	"used" : "LVM",
+	"wearout" : "N/A",
+	"health" : "UNKNOWN",
+	"gpt" : 1,
+	"journals": 1,
+	"size" : 1536000,
+	"rpm" : 0,
+	"type" : "hdd",
+	"osdid" : -1
+    },
+    "sdi" : {
+	"serial" : "SERIAL1",
+	"vendor" : "ATA",
+	"wwn" : "0x0000000000000000",
+	"devpath" : "/dev/sdi",
+	"model" : "MODEL1",
+	"used" : "LVM",
+	"wearout" : "N/A",
+	"health" : "UNKNOWN",
+	"gpt" : 1,
+	"size" : 1536000,
+	"rpm" : 0,
+	"type" : "hdd",
+	"db": 1,
+	"osdid" : -1
+    },
+    "sdj" : {
+	"serial" : "SERIAL1",
+	"vendor" : "ATA",
+	"wwn" : "0x0000000000000000",
+	"devpath" : "/dev/sdj",
+	"model" : "MODEL1",
+	"used" : "LVM",
+	"wearout" : "N/A",
+	"health" : "UNKNOWN",
+	"gpt" : 1,
+	"size" : 1536000,
+	"rpm" : 0,
+	"bluestore": 0,
+	"type" : "hdd",
+	"osdid" : 2
     }
 }
diff --git a/test/disk_tests/usages/lvs b/test/disk_tests/usages/lvs
new file mode 100644
index 0000000..b3fad43
--- /dev/null
+++ b/test/disk_tests/usages/lvs
@@ -0,0 +1,4 @@
+/dev/sdg(0);osd-block-01234;ceph.osd_id=1
+/dev/sdh(0);osd-journal-01234;ceph.osd_id=1
+/dev/sdi(0);osd-db-01234;ceph.osd_id=1
+/dev/sdj(0);osd-data-01234;ceph.osd_id=2
diff --git a/test/disk_tests/usages/pvs b/test/disk_tests/usages/pvs
index ab3ff75..0df5080 100644
--- a/test/disk_tests/usages/pvs
+++ b/test/disk_tests/usages/pvs
@@ -1 +1,5 @@
-  /dev/sdb 
+  /dev/sdb
+  /dev/sdg
+  /dev/sdh
+  /dev/sdi
+  /dev/sdj
diff --git a/test/disk_tests/usages/sdg/device/vendor b/test/disk_tests/usages/sdg/device/vendor
new file mode 100644
index 0000000..531030d
--- /dev/null
+++ b/test/disk_tests/usages/sdg/device/vendor
@@ -0,0 +1 @@
+ATA
diff --git a/test/disk_tests/usages/sdg/queue/rotational b/test/disk_tests/usages/sdg/queue/rotational
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/test/disk_tests/usages/sdg/queue/rotational
@@ -0,0 +1 @@
+1
diff --git a/test/disk_tests/usages/sdg/size b/test/disk_tests/usages/sdg/size
new file mode 100644
index 0000000..13de30f
--- /dev/null
+++ b/test/disk_tests/usages/sdg/size
@@ -0,0 +1 @@
+3000
diff --git a/test/disk_tests/usages/sdg_udevadm b/test/disk_tests/usages/sdg_udevadm
new file mode 100644
index 0000000..6d40d35
--- /dev/null
+++ b/test/disk_tests/usages/sdg_udevadm
@@ -0,0 +1,12 @@
+E: DEVNAME=/dev/sdg
+E: DEVTYPE=disk
+E: ID_ATA_ROTATION_RATE_RPM=0
+E: ID_BUS=ata
+E: ID_MODEL=MODEL1
+E: ID_PART_TABLE_TYPE=gpt
+E: ID_PART_TABLE_UUID=8417b93f-eff9-4e8f-8d84-dc2e77fc07a2
+E: ID_SERIAL=SERIAL1
+E: ID_SERIAL_SHORT=SERIAL1
+E: ID_TYPE=disk
+E: ID_WWN=0x0000000000000000
+E: ID_WWN_WITH_EXTENSION=0x0000000000000000
diff --git a/test/disk_tests/usages/sdh/device/vendor b/test/disk_tests/usages/sdh/device/vendor
new file mode 100644
index 0000000..531030d
--- /dev/null
+++ b/test/disk_tests/usages/sdh/device/vendor
@@ -0,0 +1 @@
+ATA
diff --git a/test/disk_tests/usages/sdh/queue/rotational b/test/disk_tests/usages/sdh/queue/rotational
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/test/disk_tests/usages/sdh/queue/rotational
@@ -0,0 +1 @@
+1
diff --git a/test/disk_tests/usages/sdh/size b/test/disk_tests/usages/sdh/size
new file mode 100644
index 0000000..13de30f
--- /dev/null
+++ b/test/disk_tests/usages/sdh/size
@@ -0,0 +1 @@
+3000
diff --git a/test/disk_tests/usages/sdh_udevadm b/test/disk_tests/usages/sdh_udevadm
new file mode 100644
index 0000000..3ff1a9e
--- /dev/null
+++ b/test/disk_tests/usages/sdh_udevadm
@@ -0,0 +1,12 @@
+E: DEVNAME=/dev/sdh
+E: DEVTYPE=disk
+E: ID_ATA_ROTATION_RATE_RPM=0
+E: ID_BUS=ata
+E: ID_MODEL=MODEL1
+E: ID_PART_TABLE_TYPE=gpt
+E: ID_PART_TABLE_UUID=8417b93f-eff9-4e8f-8d84-dc2e77fc07a2
+E: ID_SERIAL=SERIAL1
+E: ID_SERIAL_SHORT=SERIAL1
+E: ID_TYPE=disk
+E: ID_WWN=0x0000000000000000
+E: ID_WWN_WITH_EXTENSION=0x0000000000000000
diff --git a/test/disk_tests/usages/sdi/device/vendor b/test/disk_tests/usages/sdi/device/vendor
new file mode 100644
index 0000000..531030d
--- /dev/null
+++ b/test/disk_tests/usages/sdi/device/vendor
@@ -0,0 +1 @@
+ATA
diff --git a/test/disk_tests/usages/sdi/queue/rotational b/test/disk_tests/usages/sdi/queue/rotational
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/test/disk_tests/usages/sdi/queue/rotational
@@ -0,0 +1 @@
+1
diff --git a/test/disk_tests/usages/sdi/size b/test/disk_tests/usages/sdi/size
new file mode 100644
index 0000000..13de30f
--- /dev/null
+++ b/test/disk_tests/usages/sdi/size
@@ -0,0 +1 @@
+3000
diff --git a/test/disk_tests/usages/sdi_udevadm b/test/disk_tests/usages/sdi_udevadm
new file mode 100644
index 0000000..a9eae5e
--- /dev/null
+++ b/test/disk_tests/usages/sdi_udevadm
@@ -0,0 +1,12 @@
+E: DEVNAME=/dev/sdi
+E: DEVTYPE=disk
+E: ID_ATA_ROTATION_RATE_RPM=0
+E: ID_BUS=ata
+E: ID_MODEL=MODEL1
+E: ID_PART_TABLE_TYPE=gpt
+E: ID_PART_TABLE_UUID=8417b93f-eff9-4e8f-8d84-dc2e77fc07a2
+E: ID_SERIAL=SERIAL1
+E: ID_SERIAL_SHORT=SERIAL1
+E: ID_TYPE=disk
+E: ID_WWN=0x0000000000000000
+E: ID_WWN_WITH_EXTENSION=0x0000000000000000
diff --git a/test/disk_tests/usages/sdj/device/vendor b/test/disk_tests/usages/sdj/device/vendor
new file mode 100644
index 0000000..531030d
--- /dev/null
+++ b/test/disk_tests/usages/sdj/device/vendor
@@ -0,0 +1 @@
+ATA
diff --git a/test/disk_tests/usages/sdj/queue/rotational b/test/disk_tests/usages/sdj/queue/rotational
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/test/disk_tests/usages/sdj/queue/rotational
@@ -0,0 +1 @@
+1
diff --git a/test/disk_tests/usages/sdj/size b/test/disk_tests/usages/sdj/size
new file mode 100644
index 0000000..13de30f
--- /dev/null
+++ b/test/disk_tests/usages/sdj/size
@@ -0,0 +1 @@
+3000
diff --git a/test/disk_tests/usages/sdj_udevadm b/test/disk_tests/usages/sdj_udevadm
new file mode 100644
index 0000000..39d0cf3
--- /dev/null
+++ b/test/disk_tests/usages/sdj_udevadm
@@ -0,0 +1,12 @@
+E: DEVNAME=/dev/sdj
+E: DEVTYPE=disk
+E: ID_ATA_ROTATION_RATE_RPM=0
+E: ID_BUS=ata
+E: ID_MODEL=MODEL1
+E: ID_PART_TABLE_TYPE=gpt
+E: ID_PART_TABLE_UUID=8417b93f-eff9-4e8f-8d84-dc2e77fc07a2
+E: ID_SERIAL=SERIAL1
+E: ID_SERIAL_SHORT=SERIAL1
+E: ID_TYPE=disk
+E: ID_WWN=0x0000000000000000
+E: ID_WWN_WITH_EXTENSION=0x0000000000000000
diff --git a/test/disklist_test.pm b/test/disklist_test.pm
index 902563c..d9781ee 100644
--- a/test/disklist_test.pm
+++ b/test/disklist_test.pm
@@ -52,6 +52,8 @@ sub mocked_run_command {
 	} elsif ($cmd->[0] =~ m/pvs/i) {
 	    # simulate lvs output
 	    @$outputlines = split(/\n/, read_test_file('pvs'));
+	} elsif ($cmd->[0] =~ m/lvs/i) {
+	    @$outputlines = split(/\n/, read_test_file('lvs'));
 	} else {
 	    print "unexpected run_command call: '@$cmd', aborting\n";
 	    die;
-- 
2.11.0





More information about the pve-devel mailing list