[pve-devel] [PATCH] add virtio-vhost-user nic support
Alexandre Derumier
aderumier at odiso.com
Tue May 3 17:05:38 CEST 2016
vm config:
net0: virtio-vhost-user=66:32:37:61:66:64,bridge=vmbr0,tag=10
numa: 1
virtio-vhost-user is userland only, so no tap interface
Numa need to be enabled, and all vm memory is mapped to hugepages
The qemu netdev script/downscript don't work with vhost-user
System configuration:
hugepages need to be mounted. (can be 2MB or 1GB Hupages)
/etc/fstab
----------
hugetlbfs /dev/hugepages hugetlbfs pagesize=2048k 0 0
sysctl
------
echo 2048 > /proc/sys/vm/nr_hugepages
Openvswitch need to have dpdk enabled
/etc/default/openvswitch-switch
--------------------------------
DPDK_OPTS='--dpdk -c 0x1 -n 4'
ovs switch need to have datapath_type=netdev
/etc/network/interfaces
-----------------------
auto vmbr0
iface vmbr10 inet manual
ovs_type OVSBridge
ovs_ports eth0
ovs_extra set bridge vmbr10 datapath_type=netdev
Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
PVE/QemuServer.pm | 108 +++++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 83 insertions(+), 25 deletions(-)
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 86c2f32..9070dec 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -564,7 +564,7 @@ for (my $i = 0; $i < $MAX_NUMA; $i++) {
my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
- 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
+ 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em', 'virtio-vhost-user'];
my $nic_model_list_txt = join(' ', sort @$nic_model_list);
my $net_fmt = {
@@ -1565,9 +1565,9 @@ sub print_netdevice_full {
my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
my $device = $net->{model};
- if ($net->{model} eq 'virtio') {
+ if ($net->{model} =~ m/^virtio/) {
$device = 'virtio-net-pci';
- };
+ };
my $pciaddr = print_pci_addr("$netid", $bridges);
my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
@@ -1622,7 +1622,12 @@ sub print_netdev_full {
my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
if ($net->{bridge}) {
- $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
+ if ($net->{model} eq 'virtio-vhost-user') {
+ $netdev = "type=vhost-user,id=$netid,chardev=charvhostuser$i,vhostforce";
+ } else {
+ $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
+ }
+
} else {
$netdev = "type=user,id=$netid,hostname=$vmname";
}
@@ -3122,6 +3127,36 @@ sub config_to_command {
push @$cmd, '-cpu', $cpu;
+ my $use_vhost_user = undef;
+
+ for (my $i = 0; $i < $MAX_NETS; $i++) {
+ next if !$conf->{"net$i"};
+ my $d = parse_net($conf->{"net$i"});
+ next if !$d;
+
+ $use_virtio = 1 if $d->{model} eq 'virtio';
+ $use_vhost_user = 1 if $d->{model} eq 'virtio-vhost-user';
+
+ if ($bootindex_hash->{n}) {
+ $d->{bootindex} = $bootindex_hash->{n};
+ $bootindex_hash->{n} += 1;
+ }
+
+ if ($d->{model} eq 'virtio-vhost-user') {
+ my $vhostuser = "vhost-user".$vmid."i".$i;
+ push @$devices, '-chardev', "socket,id=charvhostuser$i,path=/var/run/openvswitch/$vhostuser";
+
+ }
+ my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
+ push @$devices, '-netdev', $netdevfull;
+
+ my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
+ push @$devices, '-device', $netdevicefull;
+ }
+
+ die "Numa need to be enabled ot use vhost-user nics" if $use_vhost_user && !$conf->{numa};
+
+
my $memory = $conf->{memory} || $defaults->{memory};
my $static_memory = 0;
my $dimm_memory = 0;
@@ -3151,7 +3186,14 @@ sub config_to_command {
die "missing numa node$i memory value\n" if !$numa->{memory};
my $numa_memory = $numa->{memory};
$numa_totalmemory += $numa_memory;
- my $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
+
+ my $numa_object = '';
+
+ if ($use_vhost_user) {
+ $numa_object = "memory-backend-file,id=ram-node$i,size=${numa_memory}M,mem-path=/dev/hugepages,share=on";
+ } else {
+ $numa_object = "memory-backend-ram,id=ram-node$i,size=${numa_memory}M";
+ }
# cpus
my $cpulists = $numa->{cpus};
@@ -3201,10 +3243,19 @@ sub config_to_command {
my $cpus = $cpustart;
$cpus .= "-$cpuend" if $cpuend;
- push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
+ my $numa_object = '';
+ if ($use_vhost_user) {
+ $numa_object = "memory-backend-file,size=$numa_memory,id=ram-node$i,mem-path=/dev/hugepages,share=on";
+ } else {
+ $numa_object = "memory-backend-ram,size=$numa_memory,id=ram-node$i";
+ }
+
+ push @$cmd, '-object', $numa_object;
push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
}
}
+
+ push @$devices, '-mem-prealloc' if $use_vhost_user;
}
if ($hotplug_features->{memory}) {
@@ -3352,25 +3403,6 @@ sub config_to_command {
push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
});
- for (my $i = 0; $i < $MAX_NETS; $i++) {
- next if !$conf->{"net$i"};
- my $d = parse_net($conf->{"net$i"});
- next if !$d;
-
- $use_virtio = 1 if $d->{model} eq 'virtio';
-
- if ($bootindex_hash->{n}) {
- $d->{bootindex} = $bootindex_hash->{n};
- $bootindex_hash->{n} += 1;
- }
-
- my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
- push @$devices, '-netdev', $netdevfull;
-
- my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
- push @$devices, '-device', $netdevicefull;
- }
-
if (!$q35) {
# add pci bridges
if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
@@ -4571,6 +4603,21 @@ sub vm_start {
}
}
+ #ovs vhost-user
+ for (my $i = 0; $i < $MAX_NETS; $i++) {
+ next if !$conf->{"net$i"};
+ my $d = parse_net($conf->{"net$i"});
+ next if !$d;
+ if ($d->{model} eq 'virtio-vhost-user') {
+ my $interface = "vhost-user".$vmid."i".$i;
+ #remove old uncleaned port
+ if(-e "/var/run/openvswitch/$interface") {
+ run_command("/usr/bin/ovs-vsctl del-port $interface", outfunc => sub {}, errfunc => sub {});
+ }
+ PVE::Network::ovs_bridge_add_port($d->{bridge}, $interface, $d->{vlan}, 'dpdkvhostuser', $d->{trunks});
+ }
+ }
+
PVE::Storage::activate_volumes($storecfg, $vollist);
if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
@@ -4746,6 +4793,17 @@ sub vm_stop_cleanup {
PVE::Storage::deactivate_volumes($storecfg, $vollist);
}
+ #ovs vhost-user
+ for (my $i = 0; $i < $MAX_NETS; $i++) {
+ next if !$conf->{"net$i"};
+ my $d = parse_net($conf->{"net$i"});
+ next if !$d;
+ if ($d->{model} eq 'virtio-vhost-user') {
+ my $ovsintport = "vhost-user".$vmid."i".$i;
+ run_command("/usr/bin/ovs-vsctl del-port $ovsintport", outfunc => sub {}, errfunc => sub {});
+ }
+ }
+
foreach my $ext (qw(mon qmp pid vnc qga)) {
unlink "/var/run/qemu-server/${vmid}.$ext";
}
--
2.1.4
More information about the pve-devel
mailing list