[pve-devel] [PATCH] memory hotplug patch v7

Alexandre Derumier aderumier at odiso.com
Wed Jan 21 15:00:32 CET 2015


This patch allow to hotplug memory dimm modules
though a new option : dimm_memory

The dimm modules are generated from a map

     dimmid       size  dimm_memory
     dimm0        128        128     100.00 0
     dimm1        128        256      50.00 1
     dimm2        128        384      33.33 2
     dimm3        128        512      25.00 3
     dimm4        128        640      20.00 0
     dimm5        128        768      16.67 1
     dimm6        128        896      14.29 2
     dimm7        128       1024      12.50 3
     dimm8        128       1152      11.11 0
     dimm9        128       1280      10.00 1
    dimm10        128       1408       9.09 2
    dimm11        128       1536       8.33 3
    dimm12        128       1664       7.69 0
    dimm13        128       1792       7.14 1
    dimm14        128       1920       6.67 2
    dimm15        128       2048       6.25 3
    dimm16        128       2176       5.88 0
    dimm17        128       2304       5.56 1
    dimm18        128       2432       5.26 2
    dimm19        128       2560       5.00 3
    dimm20        128       2688       4.76
    ....
   dimm250      16384     962560       1.70 2
   dimm251      16384     978944       1.67 3
   dimm252      16384     995328       1.65 0
   dimm253      16384    1011712       1.62 1
   dimm254      16384    1028096       1.59 2
   dimm255      16384    1044480       1.57 3

max dimm_memory size is 1TB.
If the dimm_memory value is not aligned on memory module, we align the dimm_memory on the next module.

vmid.conf
---------
memory: 1024
maxmemory: 8192
dimm_memory: 1028

total running memory is : memory + dimm_memory.

dimm_memory is not seen by os at start, so need to have "memory" enough big to load kernel

hotplug
----
qm set <vmid> -dimm_memory X    (where X is bigger than current value)

unplug (not yet implemented in qemu)
------
qm set <vmid> -delete dimm_memory
qm set <vmid> -dimm_memory X  (where X is lower than current value)

linux guest
-----------
-acpi hotplug module should be loaded in guest
-need a recent kernel. (tested with 3.10)

can be enable automaticaly, adding:

/lib/udev/rules.d/80-hotplug-cpu-mem.rules
SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", \
 ATTR{online}="1"

SUBSYSTEM=="memory", ACTION=="add", TEST=="state", ATTR{state}=="offline", \
 ATTR{state}="online"

windows guest
-------------
windows guest need numa:1 (windows limitation)

tested with:

- windows 2012 standard
- windows 2008 enterprise/datacenter

Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
 PVE/QemuServer.pm |  126 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 124 insertions(+), 2 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 7cfbc6f..e9edb0f 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -210,6 +210,19 @@ my $confdesc = {
 	minimum => 16,
 	default => 512,
     },
+    dimm_memory => {
+	optional => 1,
+	type => 'integer',
+	description => "Amount of hotplugged RAM in MB",
+	minimum => 128,
+	maximum => 1044480,
+    },
+    maxmemory => {
+       optional => 1,
+       type => 'integer',
+       description => "Maximum hotpluggale memory in MB.",
+       minimum => 128,
+    },
     balloon => {
         optional => 1,
         type => 'integer',
@@ -490,6 +503,7 @@ my $MAX_HOSTPCI_DEVICES = 4;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
+my $MAX_DIMMS = 255;
 
 my $numadesc = {
     optional => 1,
@@ -2859,8 +2873,42 @@ sub config_to_command {
     # push @$cmd, '-cpu', "$cpu,enforce";
     push @$cmd, '-cpu', $cpu;
 
-    my $memory =  $conf->{memory} || $defaults->{memory};
-    push @$cmd, '-m', $memory;
+    my $memory = $conf->{memory} || $defaults->{memory};
+    my $totalmemory = $memory;
+
+    if($conf->{dimm_memory}) {
+	die "dimm_memory should be a multiple of 128!\n" if ($conf->{dimm_memory} % 128 != 0);
+
+	$totalmemory += $conf->{dimm_memory};
+
+	my $dimm_id = 0;
+	my $current_size = 0;
+	my $dimm_size = 128;
+	for (my $j = 0; $j < 8; $j++) {
+	    for (my $i = 0; $i < 32; $i++) {
+		my $name = "dimm${dimm_id}";
+		$dimm_id++;
+		$current_size += $dimm_size;
+		my $numanode = $i % $sockets;
+		push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=".int($dimm_size*1024*1024);
+		push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
+		$i = $j = 32 if $current_size >= $conf->{dimm_memory};
+		#if dimm_memory is not aligned to dimm map
+		if($current_size >= $conf->{dimm_memory}) {
+		     $conf->{dimm_memory} = $current_size;
+		     update_config_nolock($vmid, $conf, 1);
+		}
+	     }
+	     $dimm_size *= 2;
+	}
+    }
+
+    if ($conf->{maxmemory}) {
+	die "Total memory is bigger than maxmemory\n" if $totalmemory > $conf->{maxmemory};
+	push @$cmd, '-m', "size=".$memory.",slots=$MAX_DIMMS,maxmem=".$conf->{maxmemory}."M";
+    } else {
+	push @$cmd, '-m', $memory;
+    }
 
     if ($conf->{numa}) {
 
@@ -3299,6 +3347,22 @@ sub qemu_devicedel {
     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
 }
 
+sub qemu_objectadd {
+    my($vmid, $objectid, $qomtype) = @_;
+
+    vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+
+    return 1;
+}
+
+sub qemu_objectdel {
+    my($vmid, $objectid) = @_;
+
+    vm_mon_cmd($vmid, "object-del", id => $objectid);
+
+    return 1;
+}
+
 sub qemu_driveadd {
     my ($storecfg, $vmid, $device) = @_;
 
@@ -3441,6 +3505,58 @@ sub qemu_cpu_hotplug {
     }
 }
 
+sub qemu_memory_hotplug {
+    my ($vmid, $conf, $opt, $value) = @_;
+
+    return if !check_running($vmid);
+ 
+    die "maxmemory is not defined" if !$conf->{maxmemory};
+    my $dimm_memory = $conf->{dimm_memory} ? $conf->{dimm_memory} : 0;
+    die "memory unplug is not yet available" if !$value || $value < $dimm_memory;
+
+    die "dimm_memory should be a multiple of 128!\n" if ($value % 128 != 0);
+
+
+    my $totalmemory = $conf->{memory} + $value;
+    die "you cannot add more memory than maxmemory!\n" if $totalmemory > $conf->{'maxmemory'};
+
+    my $sockets = 1;
+    $sockets = $conf->{smp} if $conf->{smp};
+    $sockets = $conf->{sockets} if  $conf->{sockets};
+
+    my $dimm_id = 0;
+    my $current_size = 0;
+    my $dimm_size = 128;
+    for (my $j = 0; $j < 8; $j++) {
+        for (my $i = 0; $i < 32; $i++) {
+            my $name = "dimm${dimm_id}";
+            $dimm_id++;
+            $current_size += $dimm_size;
+	    next if $current_size <= $dimm_memory;
+            my $numanode = $i % $sockets;
+
+	    eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+	    if (my $err = $@) {
+	        eval { qemu_objectdel($vmid, "mem-$name"); };
+	        die $err;
+	    }
+
+	    eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
+	    if (my $err = $@) {
+	        eval { qemu_objectdel($vmid, "mem-$name"); };
+	        die $err;
+	    }
+	    #update conf after each succesful module hotplug
+            $conf->{$opt} = $current_size;
+            update_config_nolock($vmid, $conf, 1);
+
+            return $current_size if $current_size >= $value;
+         }
+         $dimm_size *= 2;
+    }
+
+}
+
 sub qemu_block_set_io_throttle {
     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
 
@@ -3688,6 +3804,9 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
 		vm_deviceunplug($vmid, $conf, $opt);
 		vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+	    } elsif ($opt =~ m/^dimm_memory(\d+)$/) { #dimms
+		die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+		qemu_memory_hotplug($vmid, $conf, $opt);
 	    } else {
 		die "skip\n";
 	    }
@@ -3734,6 +3853,9 @@ sub vmconfig_hotplug_pending {
 	    } elsif (valid_drivename($opt)) {
 		# some changes can be done without hotplug
 		vmconfig_update_disk($storecfg, $conf, $vmid, $opt, $value, 1);
+	    } elsif ($opt =~ m/^dimm_memory$/) { #dimms
+		die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+		$value = qemu_memory_hotplug($vmid, $conf, $opt, $value);
 	    } else {
 		die "skip\n";  # skip non-hot-pluggable options
 	    }
-- 
1.7.10.4



More information about the pve-devel mailing list