[pve-devel] [PATCH] memory hotplug patch v8

Alexandre Derumier aderumier at odiso.com
Sun Jan 25 13:55:32 CET 2015


This patch allow to hotplug memory dimm modules
though a new option : dimm_memory

The dimm modules are generated from a map

     dimmid       size  dimm_memory
     dimm0        128        128     100.00 0
     dimm1        128        256      50.00 1
     dimm2        128        384      33.33 2
     dimm3        128        512      25.00 3
     dimm4        128        640      20.00 0
     dimm5        128        768      16.67 1
     dimm6        128        896      14.29 2
     dimm7        128       1024      12.50 3
     dimm8        128       1152      11.11 0
     dimm9        128       1280      10.00 1
    dimm10        128       1408       9.09 2
    dimm11        128       1536       8.33 3
    dimm12        128       1664       7.69 0
    dimm13        128       1792       7.14 1
    dimm14        128       1920       6.67 2
    dimm15        128       2048       6.25 3
    dimm16        128       2176       5.88 0
    dimm17        128       2304       5.56 1
    dimm18        128       2432       5.26 2
    dimm19        128       2560       5.00 3
    dimm20        128       2688       4.76
    ....
   dimm250      16384     962560       1.70 2
   dimm251      16384     978944       1.67 3
   dimm252      16384     995328       1.65 0
   dimm253      16384    1011712       1.62 1
   dimm254      16384    1028096       1.59 2
   dimm255      16384    1044480       1.57 3

max dimm_memory size is 1TB.
If the dimm_memory value is not aligned on memory module, we align the dimm_memory on the next module.

vmid.conf
---------
memory: 1024
maxmemory: 1048576
numa:1

when maxmemory is enabled, the minimum memory value must be 512MB, and also numa need to be enabled.

we assign the first 512MB as static memory, splitted on each numa nodes.
The remaining memory is assigned on hotpluggable dimm devices.

The static memory need to be also 128MB aligned, to have other dimm devices aligned too.

This 128MB alignment is a linux limitation, windows can align on 2MB size.

Numa need to be aligned, as linux guest don't boot on some setup with multi sockets,
and windows need numa to be able to hotplug memory

hotplug
----
qm set <vmid> -memory X    (where X is bigger than current value)

unplug (not yet implemented in qemu)
------
qm set <vmid> -memory X  (where X is lower than current value)

linux guest
-----------
-acpi hotplug module should be loaded in guest
-need a recent kernel. (tested with 3.10)

can be enable automaticaly, adding:

/lib/udev/rules.d/80-hotplug-cpu-mem.rules
SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", \
 ATTR{online}="1"

SUBSYSTEM=="memory", ACTION=="add", TEST=="state", ATTR{state}=="offline", \
 ATTR{state}="online"

windows guest
-------------

tested with:

- windows 2012 standard
- windows 2008 enterprise/datacenter

Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
 PVE/QemuServer.pm |  136 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 131 insertions(+), 5 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9f3a2f3..197a884 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -210,6 +210,12 @@ my $confdesc = {
 	minimum => 16,
 	default => 512,
     },
+    maxmemory => {
+       optional => 1,
+       type => 'integer',
+       description => "Maximum hotpluggale memory in MB.",
+       minimum => 128,
+    },
     balloon => {
         optional => 1,
         type => 'integer',
@@ -490,6 +496,7 @@ my $MAX_HOSTPCI_DEVICES = 4;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
+my $MAX_DIMMS = 255;
 
 my $numadesc = {
     optional => 1,
@@ -2853,8 +2860,24 @@ sub config_to_command {
     # push @$cmd, '-cpu', "$cpu,enforce";
     push @$cmd, '-cpu', $cpu;
 
-    my $memory =  $conf->{memory} || $defaults->{memory};
-    push @$cmd, '-m', $memory;
+    my $memory = $conf->{memory} || $defaults->{memory};
+    my $static_memory = 0;
+    my $dimm_memory = 0;
+
+    if ($conf->{maxmemory}) {
+	die "Numa need to be enabled for memory hotplug" if !$conf->{numa};
+	die "Total memory is bigger than maxmemory" if $memory > $conf->{maxmemory};
+	die "memory should be a multiple of 128!" if ($memory % 128 != 0);
+	$static_memory = 512;
+	die "minimum memory must be $static_memory"."MB" if($memory < $static_memory);
+	$dimm_memory = $memory - $static_memory;
+	push @$cmd, '-m', "size=".$static_memory.",slots=$MAX_DIMMS,maxmem=".$conf->{maxmemory}."M";
+
+    } else {
+
+	$static_memory = $memory;
+	push @$cmd, '-m', $static_memory;
+    }
 
     if ($conf->{numa}) {
 
@@ -2904,13 +2927,13 @@ sub config_to_command {
 	    push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 	}
 
-	die "total memory for NUMA nodes must be equal to vm memory\n"
-	    if $numa_totalmemory && $numa_totalmemory != $memory;
+	die "total memory for NUMA nodes must be equal to vm static memory\n"
+	    if $numa_totalmemory && $numa_totalmemory != $static_memory;
 
 	#if no custom tology, we split memory and cores across numa nodes
 	if(!$numa_totalmemory) {
 
-	    my $numa_memory = ($memory / $sockets) . "M";
+	    my $numa_memory = ($static_memory / $sockets) . "M";
 
 	    for (my $i = 0; $i < $sockets; $i++)  {
 
@@ -2925,6 +2948,31 @@ sub config_to_command {
 	}
     }
 
+    if($conf->{maxmemory}) {
+
+	my $dimm_id = 0;
+	my $dimm_size = 128;
+	my $current_size = $static_memory;
+	for (my $j = 0; $j < 8; $j++) {
+	    for (my $i = 0; $i < 32; $i++) {
+		my $name = "dimm${dimm_id}";
+		$dimm_id++;
+		last if $current_size >= $memory;
+		my $numanode = $i % $sockets;
+		push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=$dimm_size"."M";
+		push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
+#		$i = $j = 32 if($current_size >= $memory);
+		$current_size += $dimm_size;
+		#if dimm_memory is not aligned to dimm map
+		if($current_size > $memory) {
+		    $conf->{memory} = $current_size;
+		    update_config_nolock($vmid, $conf, 1);
+		}
+	     }
+	     $dimm_size *= 2;
+	}
+    }
+
     push @$cmd, '-S' if $conf->{freeze};
 
     # set keyboard layout
@@ -3293,6 +3341,22 @@ sub qemu_devicedel {
     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
 }
 
+sub qemu_objectadd {
+    my($vmid, $objectid, $qomtype) = @_;
+
+    vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+
+    return 1;
+}
+
+sub qemu_objectdel {
+    my($vmid, $objectid) = @_;
+
+    vm_mon_cmd($vmid, "object-del", id => $objectid);
+
+    return 1;
+}
+
 sub qemu_driveadd {
     my ($storecfg, $vmid, $device) = @_;
 
@@ -3436,6 +3500,62 @@ sub qemu_cpu_hotplug {
     }
 }
 
+sub qemu_memory_hotplug {
+    my ($vmid, $conf, $defaults, $opt, $value) = @_;
+
+    return $value if !check_running($vmid);
+ 
+    die "maxmemory is not defined" if !$conf->{maxmemory};
+
+    my $memory = $conf->{memory} || $defaults->{memory};
+    $value = $defaults->{memory} if !$value; 
+    return $value if $value == $memory;
+
+    my $static_memory = 512;
+    my $dimm_memory = $memory - $static_memory;
+
+    die "memory can't be lower than $static_memory MB" if $value < $static_memory;
+    die "memory unplug is not yet available" if $value < $memory;
+    die "memory should be a multiple of 128!\n" if ($value % 128 != 0);
+    die "you cannot add more memory than maxmemory!\n" if $memory > $conf->{'maxmemory'};
+
+
+    my $sockets = 1;
+    $sockets = $conf->{sockets} if $conf->{sockets};
+
+    my $dimm_id = 0;
+    my $current_size = $static_memory;
+    my $dimm_size = 128;
+    for (my $j = 0; $j < 8; $j++) {
+	for (my $i = 0; $i < 32; $i++) {
+	    my $name = "dimm${dimm_id}";
+	    $dimm_id++;
+	    $current_size += $dimm_size;
+	    next if $current_size <= $memory;
+	    my $numanode = $i % $sockets;
+
+	    eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+	    if (my $err = $@) {
+	        eval { qemu_objectdel($vmid, "mem-$name"); };
+	        die $err;
+	    }
+
+	    eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
+	    if (my $err = $@) {
+	        eval { qemu_objectdel($vmid, "mem-$name"); };
+	        die $err;
+	    }
+	    #update conf after each succesful module hotplug
+	    $conf->{$opt} = $current_size;
+	    update_config_nolock($vmid, $conf, 1);
+
+	    return $current_size if $current_size >= $value;
+	}
+	$dimm_size *= 2;
+    }
+
+}
+
 sub qemu_block_set_io_throttle {
     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
 
@@ -3683,6 +3803,9 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
 		vm_deviceunplug($vmid, $conf, $opt);
 		vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+	    } elsif ($opt =~ m/^memory$/) { 
+		die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+		qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
 	    } else {
 		die "skip\n";
 	    }
@@ -3729,6 +3852,9 @@ sub vmconfig_hotplug_pending {
 	    } elsif (valid_drivename($opt)) {
 		# some changes can be done without hotplug
 		vmconfig_update_disk($storecfg, $conf, $vmid, $opt, $value, 1);
+	    } elsif ($opt =~ m/^memory$/) { #dimms
+		die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+		$value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
 	    } else {
 		die "skip\n";  # skip non-hot-pluggable options
 	    }
-- 
1.7.10.4



More information about the pve-devel mailing list