[pve-devel] [PATCH] memory hotplug patch v10
Alexandre Derumier
aderumier at odiso.com
Wed Jan 28 06:47:24 CET 2015
This patch allow to hotplug memory dimm modules
though a new option : dimm_memory
The dimm modules are generated from a map
dimmid size dimm_memory
dimm0 512 512 100.00 0
dimm1 512 1024 50.00 1
dimm2 512 1536 33.33 2
dimm3 512 2048 25.00 3
dimm4 512 2560 20.00 0
dimm5 512 3072 16.67 1
dimm6 512 3584 14.29 2
dimm7 512 4096 12.50 3
dimm8 512 4608 11.11 0
dimm9 512 5120 10.00 1
dimm10 512 5632 9.09 2
dimm11 512 6144 8.33 3
dimm12 512 6656 7.69 0
dimm13 512 7168 7.14 1
dimm14 512 7680 6.67 2
dimm15 512 8192 6.25 3
dimm16 512 8704 5.88 0
dimm17 512 9216 5.56 1
dimm18 512 9728 5.26 2
dimm19 512 10240 5.00 3
dimm20 512 10752 4.76 0
...
dimm241 65536 3260416 2.01 1
dimm242 65536 3325952 1.97 2
dimm243 65536 3391488 1.93 3
dimm244 65536 3457024 1.90 0
dimm245 65536 3522560 1.86 1
dimm246 65536 3588096 1.83 2
dimm247 65536 3653632 1.79 3
dimm248 65536 3719168 1.76 0
dimm249 65536 3784704 1.73 1
dimm250 65536 3850240 1.70 2
dimm251 65536 3915776 1.67 3
dimm252 65536 3981312 1.65 0
dimm253 65536 4046848 1.62 1
dimm254 65536 4112384 1.59 2
dimm255 65536 4177920 1.57 3
max dimm_memory size is 4TB, which is the current qemu limit
If the dimm_memory value is not aligned on memory module, we align the dimm_memory on the next module.
vmid.conf
---------
memory: 1024
numa:1
hotplug: memmory
when hotplug memory option is enabled, the minimum memory value must be 1GB, and also numa need to be enabled.
we assign the first 1GB as static memory, splitted on each numa nodes.
The remaining memory is assigned on hotpluggable dimm devices.
The static memory need to be also 128MB aligned, to have other dimm devices aligned too.
This 128MB alignment is a linux limitation, windows can align on 2MB size.
Numa need to be aligned, as linux guest don't boot on some setup with multi sockets,
and windows need numa to be able to hotplug memory
hotplug
----
qm set <vmid> -memory X (where X is bigger than current value)
unplug (not yet implemented in qemu)
------
qm set <vmid> -memory X (where X is lower than current value)
linux guest
-----------
-acpi hotplug module should be loaded in guest
-need a recent kernel. (tested with 3.10)
can be enable automaticaly, adding:
/lib/udev/rules.d/80-hotplug-cpu-mem.rules
SUBSYSTEM=="cpu", ACTION=="add", TEST=="online", ATTR{online}=="0", \
ATTR{online}="1"
SUBSYSTEM=="memory", ACTION=="add", TEST=="state", ATTR{state}=="offline", \
ATTR{state}="online"
windows guest
-------------
tested with:
- windows 2012 standard
- windows 2008 enterprise/datacenter
Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
PVE/QemuServer.pm | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 123 insertions(+), 5 deletions(-)
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 5a6c278..7045c14 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -490,6 +490,8 @@ my $MAX_HOSTPCI_DEVICES = 4;
my $MAX_SERIAL_PORTS = 4;
my $MAX_PARALLEL_PORTS = 3;
my $MAX_NUMA = 8;
+my $MAX_MEM = 4194304;
+my $STATICMEM = 1024;
my $numadesc = {
optional => 1,
@@ -2625,6 +2627,7 @@ sub config_to_command {
my $have_ovz = -f '/proc/vz/vestat';
my $q35 = machine_type_is_q35($conf);
+ my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
push @$cmd, '/usr/bin/kvm';
@@ -2883,8 +2886,24 @@ sub config_to_command {
# push @$cmd, '-cpu', "$cpu,enforce";
push @$cmd, '-cpu', $cpu;
- my $memory = $conf->{memory} || $defaults->{memory};
- push @$cmd, '-m', $memory;
+ my $memory = $conf->{memory} || $defaults->{memory};
+ my $static_memory = 0;
+ my $dimm_memory = 0;
+
+ if ($hotplug_features->{memory}) {
+ die "Numa need to be enabled for memory hotplug" if !$conf->{numa};
+ die "Total memory is bigger than $MAX_MEM MB" if $memory > $MAX_MEM;
+ die "memory should be a multiple of 128!" if ($memory % 128 != 0);
+ $static_memory = $STATICMEM;
+ die "minimum memory must be $static_memory"."MB" if($memory < $static_memory);
+ $dimm_memory = $memory - $static_memory;
+ push @$cmd, '-m', "size=".$static_memory.",slots=255,maxmem=".$MAX_MEM."M";
+
+ } else {
+
+ $static_memory = $memory;
+ push @$cmd, '-m', $static_memory;
+ }
if ($conf->{numa}) {
@@ -2934,13 +2953,13 @@ sub config_to_command {
push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=ram-node$i";
}
- die "total memory for NUMA nodes must be equal to vm memory\n"
- if $numa_totalmemory && $numa_totalmemory != $memory;
+ die "total memory for NUMA nodes must be equal to vm static memory\n"
+ if $numa_totalmemory && $numa_totalmemory != $static_memory;
#if no custom tology, we split memory and cores across numa nodes
if(!$numa_totalmemory) {
- my $numa_memory = ($memory / $sockets) . "M";
+ my $numa_memory = ($static_memory / $sockets) . "M";
for (my $i = 0; $i < $sockets; $i++) {
@@ -2955,6 +2974,29 @@ sub config_to_command {
}
}
+ if ($hotplug_features->{memory}) {
+ my $dimm_id = 0;
+ my $dimm_size = 512;
+ my $current_size = $static_memory;
+ for (my $j = 0; $j < 8; $j++) {
+ for (my $i = 0; $i < 32; $i++) {
+ my $name = "dimm${dimm_id}";
+ $dimm_id++;
+ last if $current_size >= $memory;
+ my $numanode = $i % $sockets;
+ push @$cmd, "-object" , "memory-backend-ram,id=mem-$name,size=$dimm_size"."M";
+ push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
+ $current_size += $dimm_size;
+ #if dimm_memory is not aligned to dimm map
+ if($current_size > $memory) {
+ $conf->{memory} = $current_size;
+ update_config_nolock($vmid, $conf, 1);
+ }
+ }
+ $dimm_size *= 2;
+ }
+ }
+
push @$cmd, '-S' if $conf->{freeze};
# set keyboard layout
@@ -3311,6 +3353,22 @@ sub qemu_devicedel {
my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
}
+sub qemu_objectadd {
+ my($vmid, $objectid, $qomtype) = @_;
+
+ vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
+
+ return 1;
+}
+
+sub qemu_objectdel {
+ my($vmid, $objectid) = @_;
+
+ vm_mon_cmd($vmid, "object-del", id => $objectid);
+
+ return 1;
+}
+
sub qemu_driveadd {
my ($storecfg, $vmid, $device) = @_;
@@ -3454,6 +3512,60 @@ sub qemu_cpu_hotplug {
}
}
+sub qemu_memory_hotplug {
+ my ($vmid, $conf, $defaults, $opt, $value) = @_;
+
+ return $value if !check_running($vmid);
+
+ my $memory = $conf->{memory} || $defaults->{memory};
+ $value = $defaults->{memory} if !$value;
+ return $value if $value == $memory;
+
+ my $static_memory = $STATICMEM;
+ my $dimm_memory = $memory - $static_memory;
+
+ die "memory can't be lower than $static_memory MB" if $value < $static_memory;
+ die "memory unplug is not yet available" if $value < $memory;
+ die "memory should be a multiple of 128!\n" if ($value % 128 != 0);
+ die "you cannot add more memory than $MAX_MEM MB!\n" if $memory > $MAX_MEM;
+
+
+ my $sockets = 1;
+ $sockets = $conf->{sockets} if $conf->{sockets};
+
+ my $dimm_id = 0;
+ my $current_size = $static_memory;
+ my $dimm_size = 512;
+ for (my $j = 0; $j < 8; $j++) {
+ for (my $i = 0; $i < 32; $i++) {
+ my $name = "dimm${dimm_id}";
+ $dimm_id++;
+ $current_size += $dimm_size;
+ next if $current_size <= $memory;
+ my $numanode = $i % $sockets;
+
+ eval { vm_mon_cmd($vmid, "object-add", 'qom-type' => "memory-backend-ram", id => "mem-$name", props => { size => int($dimm_size*1024*1024) } ) };
+ if (my $err = $@) {
+ eval { qemu_objectdel($vmid, "mem-$name"); };
+ die $err;
+ }
+
+ eval { vm_mon_cmd($vmid, "device_add", driver => "pc-dimm", id => "$name", memdev => "mem-$name", node => $numanode) };
+ if (my $err = $@) {
+ eval { qemu_objectdel($vmid, "mem-$name"); };
+ die $err;
+ }
+ #update conf after each succesful module hotplug
+ $conf->{$opt} = $current_size;
+ update_config_nolock($vmid, $conf, 1);
+
+ return $current_size if $current_size >= $value;
+ }
+ $dimm_size *= 2;
+ }
+
+}
+
sub qemu_block_set_io_throttle {
my ($vmid, $deviceid, $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr) = @_;
@@ -3701,6 +3813,9 @@ sub vmconfig_hotplug_pending {
die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
vm_deviceunplug($vmid, $conf, $opt);
vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}));
+ } elsif ($opt =~ m/^memory$/) {
+ die "skip\n" if !$hotplug_features->{memory};
+ qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
} else {
die "skip\n";
}
@@ -3749,6 +3864,9 @@ sub vmconfig_hotplug_pending {
# some changes can be done without hotplug
vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
$vmid, $opt, $value, 1);
+ } elsif ($opt =~ m/^memory$/) { #dimms
+ die "skip\n" if !$hotplug_features->{memory};
+ $value = qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
} else {
die "skip\n"; # skip non-hot-pluggable options
}
--
1.7.10.4
More information about the pve-devel
mailing list