[pve-devel] [PATCH] memory hotplug patch v9

Alexandre Derumier aderumier at odiso.com
Mon Jan 26 09:29:52 CET 2015


This patch allow to hotplug memory dimm modules
though a new option : dimm_memory

The dimm modules are generated from a map

     dimmid       size  dimm_memory
     dimm0        512        512     100.00 0
     dimm1        512       1024      50.00 1
     dimm2        512       1536      33.33 2
     dimm3        512       2048      25.00 3
     dimm4        512       2560      20.00 0
     dimm5        512       3072      16.67 1
     dimm6        512       3584      14.29 2
     dimm7        512       4096      12.50 3
     dimm8        512       4608      11.11 0
     dimm9        512       5120      10.00 1
    dimm10        512       5632       9.09 2
    dimm11        512       6144       8.33 3
    dimm12        512       6656       7.69 0
    dimm13        512       7168       7.14 1
    dimm14        512       7680       6.67 2
    dimm15        512       8192       6.25 3
    dimm16        512       8704       5.88 0
    dimm17        512       9216       5.56 1
    dimm18        512       9728       5.26 2
    dimm19        512      10240       5.00 3
    dimm20        512      10752       4.76 0
    ...

   dimm241      65536    3260416       2.01 1
   dimm242      65536    3325952       1.97 2
   dimm243      65536    3391488       1.93 3
   dimm244      65536    3457024       1.90 0
   dimm245      65536    3522560       1.86 1
   dimm246      65536    3588096       1.83 2
   dimm247      65536    3653632       1.79 3
   dimm248      65536    3719168       1.76 0
   dimm249      65536    3784704       1.73 1
   dimm250      65536    3850240       1.70 2
   dimm251      65536    3915776       1.67 3
   dimm252      65536    3981312       1.65 0
   dimm253      65536    4046848       1.62 1
   dimm254      65536    4112384       1.59 2
   dimm255      65536    4177920       1.57 3

max dimm_memory size is 4TB, which is the current qemu limit

If the dimm_memory value is not aligned on memory module, we align the dimm_memory on the next module.

vmid.conf
---------
memory: 1024
maxmemory: 4194304
numa:1

when maxmemory is enabled, the minimum memory value must be 1GB, and also numa need to be enabled.

we assign the first 1GB as static memory, splitted on each numa nodes.
The remaining memory is assigned on hotpluggable dimm devices.

The static memory need to be also 128MB aligned, to have other dimm devices aligned too.

This 128MB alignment is a linux limitation, windows can align on 2MB size.

Numa need to be aligned, as linux guest don't boot on some setup with multi sockets,
and windows need numa to be able to hotplug memory

hotplug
----
qm set <vmid> -memory X    (where X is bigger than current value)

unplug (not yet implemented in qemu)
------
qm set <vmid> -memory X  (where X is lower than current value)

linux guest
-----------
-acpi hotplug module should be loaded in guest
-need a recent kernel. (tested with 3.10)

can be enable automaticaly, adding:

/lib/udev/rules.d/80-hotplug-cpu-mem.rules
SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", \
 ATTR{online}="1"

SUBSYSTEM=="memory", ACTION=="add", TEST=="state", ATTR{state}=="offline", \
 ATTR{state}="online"

windows guest
-------------

tested with:

- windows 2012 standard
- windows 2008 enterprise/datacenter

Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
 PVE/QemuServer.pm |  136 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 131 insertions(+), 5 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9f3a2f3..fbef287 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -210,6 +210,12 @@ my $confdesc = {
 	minimum => 16,
 	default => 512,
     },
+    maxmemory => {
+       optional => 1,
+       type => 'integer',
+       description => "Maximum hotpluggale memory in MB.",
+       minimum => 1024,
+    },
     balloon => {
         optional => 1,
         type => 'integer',
@@ -490,6 +496,8 @@ my $MAX_HOSTPCI_DEVICES = 4;
 my $MAX_SERIAL_PORTS = 4;
 my $MAX_PARALLEL_PORTS = 3;
 my $MAX_NUMA = 8;
+my $MAX_DIMMS = 255;
+my $STATICMEM = 1024;
 
 my $numadesc = {
     optional => 1,
@@ -2853,8 +2861,24 @@ sub config_to_command {
     # push @$cmd, '-cpu', "$cpu,enforce";
     push @$cmd, '-cpu', $cpu;
 
-    my $memory =  $conf->{memory} || $defaults->{memory};
-    push @$cmd, '-m', $memory;
+    my $memory = $conf->{memory} || $defaults->{memory};
+    my $static_memory = 0;
+    my $dimm_memory = 0;
+
+    if ($conf->{maxmemory}) {
+	die "Numa need to be enabled for memory hotplug" if !$conf->{numa};
+	die "Total memory is bigger than maxmemory" if $memory > $conf->{maxmemory};
+	die "memory should be a multiple of 128!" if ($memory % 128 != 0);
+	$static_memory = $STATICMEM;
+	die "minimum memory must be $static_memory"."MB" if($memory < $static_memory);
+	$dimm_memory = $memory - $static_memory;
+	push @$cmd, '-m', "size=".$static_memory.",slots=$MAX_DIMMS,maxmem=".$conf->{maxmemory}."M";
+
+    } else {
+
+	$static_memory = $memory;
+	push @$cmd, '-m', $static_memory;
+    }
 
     if ($conf->{numa}) {
 
@@ -2904,13 +2928,13 @@ sub config_to_command {
 	    push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
 	}
 
-	die "total memory for NUMA nodes must be equal to vm memory\n"
-	    if $numa_totalmemory && $numa_totalmemory != $memory;
+	die "total memory for NUMA nodes must be equal to vm static memory\n"
+	    if $numa_totalmemory && $numa_totalmemory != $static_memory;
 
 	#if no custom tology, we split memory and cores across numa nodes
 	if(!$numa_totalmemory) {
 
-	    my $numa_memory = ($memory / $sockets) . "M";
+	    my $numa_memory = ($static_memory / $sockets) . "M";
 
 	    for (my $i = 0; $i < $sockets; $i++)  {
 
@@ -2925,6 +2949,30 @@ sub config_to_command {
 	}
     }
 
+    if($conf->{maxmemory}) {
+
+	my $dimm_id = 0;
+	my $dimm_size = 512;
+	my $current_size = $static_memory;
+	for (my $j = 0; $j < 8; $j++) {
+	    for (my $i = 0; $i < 32; $i++) {
+		my $name = "dimm${dimm_id}";
+		$dimm_id++;
+		last if $current_size >= $memory;
+		my $numanode = $i % $sockets;
+		push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=$dimm_size"."M";
+		push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
+		$current_size += $dimm_size;
+		#if dimm_memory is not aligned to dimm map
+		if($current_size > $memory) {
+		    $conf->{memory} = $current_size;
+		    update_config_nolock($vmid, $conf, 1);
+		}
+	     }
+	     $dimm_size *= 2;
+	}
+    }
+
     push @$cmd, '-S' if $conf->{freeze};
 
     # set keyboard layout
@@ -3293,6 +3341,22 @@ sub qemu_devicedel {
     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
 }
 
+sub qemu_objectadd {
+    my($vmid, $objectid, $qomtype) = @_;
+
+    vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+
+    return 1;
+}
+
+sub qemu_objectdel {
+    my($vmid, $objectid) = @_;
+
+    vm_mon_cmd($vmid, "object-del", id => $objectid);
+
+    return 1;
+}
+
 sub qemu_driveadd {
     my ($storecfg, $vmid, $device) = @_;
 
@@ -3436,6 +3500,62 @@ sub qemu_cpu_hotplug {
     }
 }
 
+sub qemu_memory_hotplug {
+    my ($vmid, $conf, $defaults, $opt, $value) = @_;
+
+    return $value if !check_running($vmid);
+ 
+    die "maxmemory is not defined" if !$conf->{maxmemory};
+
+    my $memory = $conf->{memory} || $defaults->{memory};
+    $value = $defaults->{memory} if !$value; 
+    return $value if $value == $memory;
+
+    my $static_memory = $STATICMEM;
+    my $dimm_memory = $memory - $static_memory;
+
+    die "memory can't be lower than $static_memory MB" if $value < $static_memory;
+    die "memory unplug is not yet available" if $value < $memory;
+    die "memory should be a multiple of 128!\n" if ($value % 128 != 0);
+    die "you cannot add more memory than maxmemory!\n" if $memory > $conf->{'maxmemory'};
+
+
+    my $sockets = 1;
+    $sockets = $conf->{sockets} if $conf->{sockets};
+
+    my $dimm_id = 0;
+    my $current_size = $static_memory;
+    my $dimm_size = 512;
+    for (my $j = 0; $j < 8; $j++) {
+	for (my $i = 0; $i < 32; $i++) {
+	    my $name = "dimm${dimm_id}";
+	    $dimm_id++;
+	    $current_size += $dimm_size;
+	    next if $current_size <= $memory;
+	    my $numanode = $i % $sockets;
+
+	    eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+	    if (my $err = $@) {
+	        eval { qemu_objectdel($vmid, "mem-$name"); };
+	        die $err;
+	    }
+
+	    eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
+	    if (my $err = $@) {
+	        eval { qemu_objectdel($vmid, "mem-$name"); };
+	        die $err;
+	    }
+	    #update conf after each succesful module hotplug
+	    $conf->{$opt} = $current_size;
+	    update_config_nolock($vmid, $conf, 1);
+
+	    return $current_size if $current_size >= $value;
+	}
+	$dimm_size *= 2;
+    }
+
+}
+
 sub qemu_block_set_io_throttle {
     my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
 
@@ -3683,6 +3803,9 @@ sub vmconfig_hotplug_pending {
 		die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
 		vm_deviceunplug($vmid, $conf, $opt);
 		vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+	    } elsif ($opt =~ m/^memory$/) { 
+		die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+		qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
 	    } else {
 		die "skip\n";
 	    }
@@ -3729,6 +3852,9 @@ sub vmconfig_hotplug_pending {
 	    } elsif (valid_drivename($opt)) {
 		# some changes can be done without hotplug
 		vmconfig_update_disk($storecfg, $conf, $vmid, $opt, $value, 1);
+	    } elsif ($opt =~ m/^memory$/) { #dimms
+		die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+		$value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
 	    } else {
 		die "skip\n";  # skip non-hot-pluggable options
 	    }
-- 
1.7.10.4




More information about the pve-devel mailing list