[pve-devel] [PATCH qemu-server v2 12/13] fix #3574: enable multi pci device mapping from config
Dominik Csapak
d.csapak at proxmox.com
Thu Aug 25 11:24:26 CEST 2022
The hardware config now supports multiple devices as a semicolon
seperated list. With this, instead of only having one device in a pci mapping,
we now have a list of which we can choose from on vm start. This way one can
dynamically start vms with a pool of (identical) pci devices without
having to manually assign the proper ids.
For that we have to change the internal representation of a parsed
device, such that we have the seperately configured paths in the mapping
in different lists (because multifunction devices still are interpreted
as single devices)
For mdev devices we now can also have multiple devices, where we simply
try to create the appropriate type on each until we either have one
created, or bail out.
Since we now have to reserve the pci ids in print_hostpci_devices, we
have to add a 'reserve' parameter to config_to_command (and chain it
through to reserve_pci_usage) so that a 'qm showcmd' does not actually
reserve any pci id (this would break when using that on running vms).
Additionally this also prevents the migration tests from failing
(they use vm_commandline which in turn uses config_to_command)
Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
---
PVE/QemuServer.pm | 43 ++++++++++++++++++++++++++++++-------
PVE/QemuServer/PCI.pm | 49 ++++++++++++++++++++++++++++++++++++-------
2 files changed, 76 insertions(+), 16 deletions(-)
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 4551e2b..ce24b19 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -3507,8 +3507,9 @@ my sub should_disable_smm {
sub config_to_command {
my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
- $pbs_backing) = @_;
+ $pbs_backing, $reserve) = @_;
+ $reserve //= 1;
my $cmd = [];
my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
my $devices = [];
@@ -3724,7 +3725,7 @@ sub config_to_command {
# host pci device passthrough
my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
- $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
+ $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder, $reserve);
# usb devices
my $usb_dev_features = {};
@@ -5623,13 +5624,30 @@ sub vm_start_nolock {
my $uuid;
for my $id (sort keys %$pci_devices) {
my $d = $pci_devices->{$id}->{device};
- for my $dev ($d->{pciid}->@*) {
- my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
- # nvidia grid needs the uuid of the mdev as qemu parameter
- if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
- $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
+ # used pci devices for non-mdev
+ if (!$d->{mdev}) {
+ for my $dev ($pci_devices->{$id}->{used}->@*) {
+ PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id);
}
+ next;
+ }
+
+ # try each configured pci device for mdevs
+ my $devs = [map { $_->{id} } map { @$_ } $d->{ids}->@*]; # flatten ids
+
+ my $info;
+ for my $dev (@$devs) {
+ $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev, $id, $d->{mdev}) };
+ warn $@ if $@;
+ last if $info; # if successful, we're done
+ }
+
+ die "could not create mediated device\n" if !defined($info);
+
+ # nvidia grid needs the uuid of the mdev as qemu parameter
+ if (!defined($uuid) && $info->{vendor} eq '10de') {
+ $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
}
}
push @$cmd, '-uuid', $uuid if defined($uuid);
@@ -5862,7 +5880,16 @@ sub vm_commandline {
my $defaults = load_defaults();
- my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
+ my $cmd = config_to_command(
+ $storecfg,
+ $vmid,
+ $conf,
+ $defaults,
+ $forcemachine,
+ $forcecpu,
+ undef,
+ 0,
+ );
return PVE::Tools::cmd2string($cmd);
}
diff --git a/PVE/QemuServer/PCI.pm b/PVE/QemuServer/PCI.pm
index 8c171f3..df8d16a 100644
--- a/PVE/QemuServer/PCI.pm
+++ b/PVE/QemuServer/PCI.pm
@@ -386,6 +386,7 @@ sub parse_hostpci {
my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
+ my $idlist = [];
if ($res->{host} !~ m/:/) {
# we have no ordinary pci id, must be a mapping
my $device = PVE::HardwareMap::find_device_on_current_node('pci', $res->{host});
@@ -396,15 +397,27 @@ sub parse_hostpci {
if (my $err = $@) {
die "PCI device mapping invalid (hardware probably changed): $err\n";
}
- $res->{host} = $device->{path};
+ $idlist = [split(/;/, $device->{path})];
+ # if we have a list of mapped devices, we want to choose the first available one
+ $res->{choose} = 1 if scalar(@$idlist > 1);
+ } else {
+ $idlist = [split(/;/, $res->{host})];
}
- my @idlist = split(/;/, $res->{host});
delete $res->{host};
- foreach my $id (@idlist) {
+ my $ignore_mdev = !$res->{choose} && scalar(@$idlist) > 1;
+
+ $res->{ids} = [];
+ foreach my $id (@$idlist) {
my $devs = PVE::SysFSTools::lspci($id);
die "no PCI device found for '$id'\n" if !scalar(@$devs);
- push @{$res->{pciid}}, @$devs;
+ $ignore_mdev = 1 if scalar(@$devs) > 1;
+ push @{$res->{ids}}, $devs;
+ }
+ # ignore mdev for multiple devices, except when from mapping
+ if ($res->{mdev} && $ignore_mdev) {
+ warn "ignoring mediated device with multifunction device\n";
+ delete $res->{mdev};
}
return $res;
}
@@ -433,11 +446,13 @@ my $print_pci_device = sub {
};
sub print_hostpci_devices {
- my ($vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder) = @_;
+ my ($vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder, $reserve) = @_;
+ $reserve //= 1;
my $kvm_off = 0;
my $gpu_passthrough = 0;
my $legacy_igd = 0;
+ my $used_pci_ids = {};
my $parsed_devices = {};
my $pciaddr;
@@ -469,7 +484,24 @@ sub print_hostpci_devices {
$pciaddr = print_pci_addr($pci_name, $bridges, $arch, $machine_type);
}
- my $pcidevices = $d->{pciid};
+ # choose devices
+ my $pcidevices = [];
+ if (!$d->{mdev}) {
+ for my $devs ($d->{ids}->@*) {
+ my $ids = [map { $_->{id} } @$devs];
+
+ if ($d->{choose}) {
+ next if grep { defined($used_pci_ids->{$_}) } @$ids; # already used
+ eval { reserve_pci_usage($ids, $vmid, 10, undef, $reserve) };
+ next if $@;
+ }
+
+ map { $used_pci_ids->{$_} = 1 } @$ids;
+ push @$pcidevices, @$devs;
+ last if $d->{choose};
+ }
+ die "could not find a free device\n" if scalar(@$pcidevices) < 1;
+ }
$parsed_devices->{$i}->{used} = $pcidevices;
my $multifunction = @$pcidevices > 1;
@@ -599,8 +631,9 @@ sub remove_pci_reservation {
}
sub reserve_pci_usage {
- my ($requested_ids, $vmid, $timeout, $pid) = @_;
+ my ($requested_ids, $vmid, $timeout, $pid, $reserve) = @_;
+ $reserve //= 1;
$requested_ids = [ $requested_ids ] if !ref($requested_ids);
return if !scalar(@$requested_ids); # do nothing for empty list
@@ -633,7 +666,7 @@ sub reserve_pci_usage {
$reservation_list->{$id}->{time} = $ctime + $timeout + 5;
}
}
- $write_pci_reservation_unlocked->($reservation_list);
+ $write_pci_reservation_unlocked->($reservation_list) if $reserve;
});
die $@ if $@;
}
--
2.30.2
More information about the pve-devel
mailing list