[pve-devel] [PATCH] add support for hugetlbfs

[Alexandre Derumier]

config
------
hugepages: 2|1024

2: enable 2M hugepages
1024 : enable 1G hugepages

1G hugepages need to be enable at boot with grub option : hugepagesz=1G,
and it's working only on new processor (>westmere for intel processor and also recent amd)

This is working with or without numa

Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
 PVE/QemuServer.pm |   56 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 53 insertions(+), 3 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 4840c73..8829077 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -210,6 +210,12 @@ my $confdesc = {
 	minimum => 16,
 	default => 512,
     },
+    hugepages => {
+	optional => 1,
+	type => 'string',
+	description => "enable manual hugetlbfs allocation",
+	enum => [qw(2 1024)],
+    },
     balloon => {
         optional => 1,
         type => 'integer',
@@ -2744,8 +2750,49 @@ sub config_to_command {
     my $memory =  $conf->{memory} || $defaults->{memory};
     push @$cmd, '-m', $memory;
 
+    my $hugepages_mountpoint = "/dev/hugepages-$conf->{hugepages}M" if $conf->{hugepages};
+
+    if($conf->{hugepages}){
+
+	#check if hugepages are available
+
+	my $hugepages_syspath = "/sys/kernel/mm/hugepages/hugepages-".($conf->{hugepages}*1024)."kB";
+	die "$conf->{hugepages}M hugepages are not available (1G hugepages need to be enable with hugepagesz=1G in grub)" if !(-d $hugepages_syspath);
+
+	#allocate hugepages if needed
+	my $hugepages_free = PVE::Tools::file_read_firstline($hugepages_syspath."/free_hugepages");
+	my $hugepages_memoryfree = $hugepages_free * $conf->{hugepages};
+	if($hugepages_memoryfree < $memory){
+	    my $hugepages_neededmemory = $memory - $hugepages_memoryfree;
+	    my $hugepages_neededpages = POSIX::ceil($hugepages_neededmemory / $conf->{hugepages});
+	    my $hugepages_current = PVE::Tools::file_read_firstline($hugepages_syspath."/nr_hugepages");
+
+	    PVE::ProcFSTools::write_proc_entry("$hugepages_syspath/nr_hugepages", $hugepages_current + $hugepages_neededpages);
+	}
+
+	#mount the hugetlbfs
+	my $fd = IO::File->new("/proc/mounts", "r") ||
+	    die "unable to open /proc/mounts - $!\n";
+	my $mounted = undef;
+
+	while (defined(my $line = <$fd>)) {
+	    my ($dev, $path, $fstype) = split(/\s+/, $line);
+	    $mounted = 1 if($fstype eq 'hugetlbfs' && $path eq $hugepages_mountpoint);
+	}
+	close($fd);
+	
+	if(!$mounted){
+            mkdir $hugepages_mountpoint if !(-d $hugepages_mountpoint);
+	    my $cmd = ['/bin/mount', '-t', 'hugetlbfs', 'none', $hugepages_mountpoint, '-o', "pagesize=$conf->{hugepages}M"];
+	    run_command($cmd, errmsg => "mount error");
+	}
+    }
+
     if ($conf->{numa}) {
 
+	my $hugepages = $hugepages_mountpoint ? ",mem-path=$hugepages_mountpoint" : "";
+	my $backend = $hugepages_mountpoint ? 'memory-backend-file,prealloc=yes' : 'memory-backend-ram';
+
 	my $numa_totalmemory = undef;
 	for (my $i = 0; $i < $MAX_NUMA; $i++) {
 	    next if !$conf->{"numa$i"};
@@ -2755,7 +2802,7 @@ sub config_to_command {
 	    die "missing numa node$i memory value\n" if !$numa->{memory};
 	    my $numa_memory = $numa->{memory};
 	    $numa_totalmemory += $numa_memory;
-	    my $numa_object = "memory-backend-ram,id=ram-node$i,size=$numa_memory"."M";
+	    my $numa_object = "$backend$hugepages,id=ram-node$i$hugepages,size=$numa_memory"."M";
 
 	    # cpus
 	    my $cpus_start = $numa->{cpus}->{start};
@@ -2806,11 +2853,14 @@ sub config_to_command {
 		my $cpuend = ($cpustart + $cores - 1) if $cores && $cores > 1;
 		my $cpus = $cpustart;
 		$cpus .= "-$cpuend" if $cpuend;
-
-		push @$cmd, '-object', "memory-backend-ram,size=$numa_memory,id=ram-node$i";
+		
+		push @$cmd, '-object', "$backend$hugepages,size=$numa_memory,id=ram-node$i";
 		push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 	    }
 	}
+    }else{
+
+	push @$cmd, '-mem-prealloc','-mem-path', $hugepages_mountpoint if $conf->{hugepages};
     }
 
     push @$cmd, '-S' if $conf->{freeze};
-- 
1.7.10.4