[pve-devel] [PATCH] memory hotplug patch v8
Alexandre Derumier
aderumier at odiso.com
Sun Jan 25 13:55:32 CET 2015
This patch allow to hotplug memory dimm modules
though a new option : dimm_memory
The dimm modules are generated from a map
dimmid size dimm_memory
dimm0 128 128 100.00 0
dimm1 128 256 50.00 1
dimm2 128 384 33.33 2
dimm3 128 512 25.00 3
dimm4 128 640 20.00 0
dimm5 128 768 16.67 1
dimm6 128 896 14.29 2
dimm7 128 1024 12.50 3
dimm8 128 1152 11.11 0
dimm9 128 1280 10.00 1
dimm10 128 1408 9.09 2
dimm11 128 1536 8.33 3
dimm12 128 1664 7.69 0
dimm13 128 1792 7.14 1
dimm14 128 1920 6.67 2
dimm15 128 2048 6.25 3
dimm16 128 2176 5.88 0
dimm17 128 2304 5.56 1
dimm18 128 2432 5.26 2
dimm19 128 2560 5.00 3
dimm20 128 2688 4.76
....
dimm250 16384 962560 1.70 2
dimm251 16384 978944 1.67 3
dimm252 16384 995328 1.65 0
dimm253 16384 1011712 1.62 1
dimm254 16384 1028096 1.59 2
dimm255 16384 1044480 1.57 3
max dimm_memory size is 1TB.
If the dimm_memory value is not aligned on memory module, we align the dimm_memory on the next module.
vmid.conf
---------
memory: 1024
maxmemory: 1048576
numa:1
when maxmemory is enabled, the minimum memory value must be 512MB, and also numa need to be enabled.
we assign the first 512MB as static memory, splitted on each numa nodes.
The remaining memory is assigned on hotpluggable dimm devices.
The static memory need to be also 128MB aligned, to have other dimm devices aligned too.
This 128MB alignment is a linux limitation, windows can align on 2MB size.
Numa need to be aligned, as linux guest don't boot on some setup with multi sockets,
and windows need numa to be able to hotplug memory
hotplug
----
qm set <vmid> -memory X (where X is bigger than current value)
unplug (not yet implemented in qemu)
------
qm set <vmid> -memory X (where X is lower than current value)
linux guest
-----------
-acpi hotplug module should be loaded in guest
-need a recent kernel. (tested with 3.10)
can be enable automaticaly, adding:
/lib/udev/rules.d/80-hotplug-cpu-mem.rules
SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", \
ATTR{online}="1"
SUBSYSTEM=="memory", ACTION=="add", TEST=="state", ATTR{state}=="offline", \
ATTR{state}="online"
windows guest
-------------
tested with:
- windows 2012 standard
- windows 2008 enterprise/datacenter
Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
PVE/QemuServer.pm | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 131 insertions(+), 5 deletions(-)
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9f3a2f3..197a884 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -210,6 +210,12 @@ my $confdesc = {
minimum => 16,
default => 512,
},
+ maxmemory => {
+ optional => 1,
+ type => 'integer',
+ description => "Maximum hotpluggale memory in MB.",
+ minimum => 128,
+ },
balloon => {
optional => 1,
type => 'integer',
@@ -490,6 +496,7 @@ my $MAX_HOSTPCI_DEVICES = 4;
my $MAX_SERIAL_PORTS = 4;
my $MAX_PARALLEL_PORTS = 3;
my $MAX_NUMA = 8;
+my $MAX_DIMMS = 255;
my $numadesc = {
optional => 1,
@@ -2853,8 +2860,24 @@ sub config_to_command {
# push @$cmd, '-cpu', "$cpu,enforce";
push @$cmd, '-cpu', $cpu;
- my $memory = $conf->{memory} || $defaults->{memory};
- push @$cmd, '-m', $memory;
+ my $memory = $conf->{memory} || $defaults->{memory};
+ my $static_memory = 0;
+ my $dimm_memory = 0;
+
+ if ($conf->{maxmemory}) {
+ die "Numa need to be enabled for memory hotplug" if !$conf->{numa};
+ die "Total memory is bigger than maxmemory" if $memory > $conf->{maxmemory};
+ die "memory should be a multiple of 128!" if ($memory % 128 != 0);
+ $static_memory = 512;
+ die "minimum memory must be $static_memory"."MB" if($memory < $static_memory);
+ $dimm_memory = $memory - $static_memory;
+ push @$cmd, '-m', "size=".$static_memory.",slots=$MAX_DIMMS,maxmem=".$conf->{maxmemory}."M";
+
+ } else {
+
+ $static_memory = $memory;
+ push @$cmd, '-m', $static_memory;
+ }
if ($conf->{numa}) {
@@ -2904,13 +2927,13 @@ sub config_to_command {
push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
}
- die "total memory for NUMA nodes must be equal to vm memory\n"
- if $numa_totalmemory && $numa_totalmemory != $memory;
+ die "total memory for NUMA nodes must be equal to vm static memory\n"
+ if $numa_totalmemory && $numa_totalmemory != $static_memory;
#if no custom tology, we split memory and cores across numa nodes
if(!$numa_totalmemory) {
- my $numa_memory = ($memory / $sockets) . "M";
+ my $numa_memory = ($static_memory / $sockets) . "M";
for (my $i = 0; $i < $sockets; $i++) {
@@ -2925,6 +2948,31 @@ sub config_to_command {
}
}
+ if($conf->{maxmemory}) {
+
+ my $dimm_id = 0;
+ my $dimm_size = 128;
+ my $current_size = $static_memory;
+ for (my $j = 0; $j < 8; $j++) {
+ for (my $i = 0; $i < 32; $i++) {
+ my $name = "dimm${dimm_id}";
+ $dimm_id++;
+ last if $current_size >= $memory;
+ my $numanode = $i % $sockets;
+ push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=$dimm_size"."M";
+ push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
+# $i = $j = 32 if($current_size >= $memory);
+ $current_size += $dimm_size;
+ #if dimm_memory is not aligned to dimm map
+ if($current_size > $memory) {
+ $conf->{memory} = $current_size;
+ update_config_nolock($vmid, $conf, 1);
+ }
+ }
+ $dimm_size *= 2;
+ }
+ }
+
push @$cmd, '-S' if $conf->{freeze};
# set keyboard layout
@@ -3293,6 +3341,22 @@ sub qemu_devicedel {
my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
}
+sub qemu_objectadd {
+ my($vmid, $objectid, $qomtype) = @_;
+
+ vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+
+ return 1;
+}
+
+sub qemu_objectdel {
+ my($vmid, $objectid) = @_;
+
+ vm_mon_cmd($vmid, "object-del", id => $objectid);
+
+ return 1;
+}
+
sub qemu_driveadd {
my ($storecfg, $vmid, $device) = @_;
@@ -3436,6 +3500,62 @@ sub qemu_cpu_hotplug {
}
}
+sub qemu_memory_hotplug {
+ my ($vmid, $conf, $defaults, $opt, $value) = @_;
+
+ return $value if !check_running($vmid);
+
+ die "maxmemory is not defined" if !$conf->{maxmemory};
+
+ my $memory = $conf->{memory} || $defaults->{memory};
+ $value = $defaults->{memory} if !$value;
+ return $value if $value == $memory;
+
+ my $static_memory = 512;
+ my $dimm_memory = $memory - $static_memory;
+
+ die "memory can't be lower than $static_memory MB" if $value < $static_memory;
+ die "memory unplug is not yet available" if $value < $memory;
+ die "memory should be a multiple of 128!\n" if ($value % 128 != 0);
+ die "you cannot add more memory than maxmemory!\n" if $memory > $conf->{'maxmemory'};
+
+
+ my $sockets = 1;
+ $sockets = $conf->{sockets} if $conf->{sockets};
+
+ my $dimm_id = 0;
+ my $current_size = $static_memory;
+ my $dimm_size = 128;
+ for (my $j = 0; $j < 8; $j++) {
+ for (my $i = 0; $i < 32; $i++) {
+ my $name = "dimm${dimm_id}";
+ $dimm_id++;
+ $current_size += $dimm_size;
+ next if $current_size <= $memory;
+ my $numanode = $i % $sockets;
+
+ eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+ if (my $err = $@) {
+ eval { qemu_objectdel($vmid, "mem-$name"); };
+ die $err;
+ }
+
+ eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
+ if (my $err = $@) {
+ eval { qemu_objectdel($vmid, "mem-$name"); };
+ die $err;
+ }
+ #update conf after each succesful module hotplug
+ $conf->{$opt} = $current_size;
+ update_config_nolock($vmid, $conf, 1);
+
+ return $current_size if $current_size >= $value;
+ }
+ $dimm_size *= 2;
+ }
+
+}
+
sub qemu_block_set_io_throttle {
my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
@@ -3683,6 +3803,9 @@ sub vmconfig_hotplug_pending {
die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
vm_deviceunplug($vmid, $conf, $opt);
vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+ } elsif ($opt =~ m/^memory$/) {
+ die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+ qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
} else {
die "skip\n";
}
@@ -3729,6 +3852,9 @@ sub vmconfig_hotplug_pending {
} elsif (valid_drivename($opt)) {
# some changes can be done without hotplug
vmconfig_update_disk($storecfg, $conf, $vmid, $opt, $value, 1);
+ } elsif ($opt =~ m/^memory$/) { #dimms
+ die "skip\n" if !$conf->{maxmemory} && !$conf->{hotplug};
+ $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
} else {
die "skip\n"; # skip non-hot-pluggable options
}
--
1.7.10.4
More information about the pve-devel
mailing list