[pve-devel] [PATCH v4 qemu-server] Add API for disk & VM import
Dominic Jäger
d.jaeger at proxmox.com
Fri Feb 5 11:04:41 CET 2021
Extend qm importdisk/importovf functionality to the API.
qm can be adapted to use this later.
Signed-off-by: Dominic Jäger <d.jaeger at proxmox.com>
---
Biggest v3->v4 changes:
* New code instead of bloating update_vm_api
* Don't change anything in the existing schema, use new parameter "diskimages"
* Because this can happen later:
- Only root can use this
- Don't touch qm (yet)
PVE/API2/Qemu.pm | 375 +++++++++++++++++++++++++++++++++++++++++-
PVE/QemuServer.pm | 16 +-
PVE/QemuServer/OVF.pm | 10 +-
3 files changed, 394 insertions(+), 7 deletions(-)
diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index 3571f5e..1ed763b 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -45,7 +45,6 @@ BEGIN {
}
}
-use Data::Dumper; # fixme: remove
use base qw(PVE::RESTHandler);
@@ -4325,4 +4324,378 @@ __PACKAGE__->register_method({
return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
}});
+# Raise exception if $format is not supported by $storageid
+my $check_format_is_supported = sub {
+ my ($format, $storageid) = @_;
+
+ return if !$format;
+
+ my $store_conf = PVE::Storage::config();
+ my (undef, $valid_formats) = PVE::Storage::storage_default_format($store_conf, $storageid);
+ my $supported = grep { $_ eq $format } @$valid_formats;
+
+ if (!$supported) {
+ raise_param_exc({format => "$format is not supported on storage $storageid"});
+ }
+};
+
+# paths are returned as is
+# volids are returned as paths
+#
+# Also checks if $original actually exists
+my $convert_to_path = sub {
+ my ($original) = @_;
+ my $volid_as_path = eval { # Nonempty iff $original_source is a volid
+ PVE::Storage::path(PVE::Storage::config(), $original);
+ };
+ my $result = $volid_as_path || $original ;
+ if (!-e $result) {
+ die "Could not import because source '$original' does not exist!";
+ }
+ return $result;
+};
+
+# vmid ... target VM ID
+# source ... absolute path of the source image (volid must be converted before)
+# storage ... target storage for the disk image
+# format ... target format for the disk image (optional)
+#
+# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2)
+my $import_disk_image = sub {
+ my ($param) = @_;
+ my $vmid = $param->{vmid};
+ my $requested_format = $param->{format};
+ my $storage = $param->{storage};
+ my $source = $param->{source};
+
+ my $vm_conf = PVE::QemuConfig->load_config($vmid);
+ my $store_conf = PVE::Storage::config();
+ if (!$source) {
+ die "It is necessary to pass the source parameter";
+ }
+ if ($source !~ m!^/!) {
+ die "source must be an absolute path but is $source";
+ }
+ if (!-e $source) {
+ die "Could not import because source $source does not exist!";
+ }
+ if (!$storage) {
+ die "It is necessary to pass the storage parameter";
+ }
+
+ print "Importing disk image '$source'...\n";
+
+ my $src_size = PVE::Storage::file_size_info($source);
+ if (!defined($src_size)) {
+ die "Could not get file size of $source";
+ } elsif (!$src_size) {
+ die "Size of file $source is 0";
+ } elsif ($src_size==1) {
+ die "Cannot import a directory";
+ }
+
+ $check_format_is_supported->($requested_format, $storage);
+
+ my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
+ $store_conf, $storage, undef, $requested_format);
+ my $dst_volid = PVE::Storage::vdisk_alloc($store_conf, $storage,
+ $vmid, $dst_format, undef, $src_size / 1024);
+
+ eval {
+ local $SIG{INT} =
+ local $SIG{TERM} =
+ local $SIG{QUIT} =
+ local $SIG{HUP} =
+ local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
+
+ my $zeroinit = PVE::Storage::volume_has_feature($store_conf,
+ 'sparseinit', $dst_volid);
+
+ PVE::Storage::activate_volumes($store_conf, [$dst_volid]);
+ PVE::QemuServer::qemu_img_convert($source, $dst_volid,
+ $src_size, undef, $zeroinit);
+ PVE::Storage::deactivate_volumes($store_conf, [$dst_volid]);
+
+ };
+ if (my $err = $@) {
+ eval { PVE::Storage::vdisk_free($store_conf, $dst_volid) };
+ warn "Cleanup of $dst_volid failed: $@ \n" if $@;
+
+ die "Importing disk '$source' failed: $err\n" if $err;
+ }
+
+ return $dst_volid;
+};
+
+__PACKAGE__->register_method ({
+ name => 'importdisk',
+ path => '{vmid}/importdisk',
+ method => 'POST',
+ protected => 1, # for worker upid file
+ proxyto => 'node',
+ description => "Import an external disk image into a VM. The image format ".
+ "has to be supported by qemu-img.",
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid',
+ {completion => \&PVE::QemuServer::complete_vmid}),
+ source => {
+ description => "Disk image to import. Can be a volid ".
+ "(local-lvm:vm-104-disk-0), an image on a PVE storage ".
+ "(local:104/toImport.raw) or (for root only) an absolute ".
+ "path on the server.",
+ type => 'string',
+ format => 'pve-volume-id-or-absolute-path',
+ },
+ device => {
+ type => 'string',
+ description => "Bus/Device type of the new disk (e.g. 'ide0', ".
+ "'scsi2'). Will add the image as unused disk if omitted.",
+ enum => [PVE::QemuServer::Drive::valid_drive_names()],
+ optional => 1,
+ },
+ device_options => {
+ type => 'string',
+ description => "Options to set for the new disk ".
+ "(e.g. 'discard=on,backup=0')",
+ optional => 1,
+ requires => 'device',
+ },
+ storage => get_standard_option('pve-storage-id', {
+ description => "The storage to which the image will be imported to.",
+ completion => \&PVE::QemuServer::complete_storage,
+ }),
+ format => {
+ type => 'string',
+ description => 'Target format.',
+ enum => [ 'raw', 'qcow2', 'vmdk' ],
+ optional => 1,
+ },
+ digest => get_standard_option('pve-config-digest'),
+ },
+ },
+ returns => { type => 'string'},
+ code => sub {
+ my ($param) = @_;
+ my $vmid = extract_param($param, 'vmid');
+ my $node = extract_param($param, 'node');
+ my $original_source = extract_param($param, 'source');
+ my $digest = extract_param($param, 'digest');
+ my $device_options = extract_param($param, 'device_options');
+ my $device = extract_param($param, 'device');
+ my $storecfg = PVE::Storage::config();
+ my $storeid = extract_param($param, 'storage');
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $format_explicit = extract_param($param, 'format');
+ my $format_device_option;
+ if ($device_options) {
+ $device_options =~ m/format=([^,]*)/;
+ $format_device_option = $1;
+ if ($format_explicit && $format_device_option) {
+ raise_param_exc({format => "Disk format may be specified only once!"});
+ }
+ }
+ my $format = $format_explicit || $format_device_option;
+ $check_format_is_supported->($format, $storeid);
+
+ my $locked = sub {
+ my $conf = PVE::QemuConfig->load_config($vmid);
+ PVE::Tools::assert_if_modified($conf->{digest}, $digest);
+
+ if ($device && $conf->{$device}) {
+ die "Could not import because device $device is already in ".
+ "use in VM $vmid. Choose a different device!";
+ }
+
+ my $imported_volid = $import_disk_image->({
+ vmid => $vmid,
+ source => $convert_to_path->($original_source),
+ storage => $storeid,
+ format => $format,
+ });
+
+ my $volid = $imported_volid;
+ if ($device) {
+ # Attach with specified options
+ $volid .= ",${device_options}" if $device_options;
+ } else {
+ # Add as unused to config
+ $device = PVE::QemuConfig->add_unused_volume($conf, $imported_volid);
+ }
+ $update_vm_api->({
+ node => $node,
+ vmid => $vmid,
+ $device => $volid,
+ });
+ };
+ my $worker = sub {
+ PVE::QemuConfig->lock_config_full($vmid, 1, $locked);
+ };
+ return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker);
+ }});
+
+__PACKAGE__->register_method({
+ name => 'importvm',
+ path => '{vmid}/importvm',
+ method => 'POST',
+ description => "Import a VM from existing disk images.",
+ protected => 1,
+ proxyto => 'node',
+ parameters => {
+ additionalProperties => 0,
+ properties => PVE::QemuServer::json_config_properties(
+ {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }),
+ diskimages => {
+ description => "Mapping of devices to disk images." .
+ "For example, scsi0:/mnt/nfs/image1.vmdk,scsi1:/mnt/nfs/image2",
+ type => 'string',
+ },
+ start => {
+ optional => 1,
+ type => 'boolean',
+ default => 0,
+ description => "Start VM after it was imported successfully.",
+ },
+ }),
+ },
+ returns => {
+ type => 'string',
+ },
+ code => sub {
+ my ($param) = @_;
+ my $node = extract_param($param, 'node');
+ my $vmid = extract_param($param, 'vmid');
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+ my $storecfg = PVE::Storage::config();
+
+ PVE::Cluster::check_cfs_quorum();
+
+ my $diskimages_string = extract_param($param, 'diskimages');
+ my @diskimage_pairs = split(',', $diskimages_string);
+
+ my $use_import = sub {
+ my ($opt) = @_;
+ return 0 if $opt eq 'efidisk0';
+ return PVE::QemuServer::Drive::is_valid_drivename($opt);
+ };
+
+ my $msg = "There must be exactly as many devices specified as there " .
+ " are devices in the diskimage parameter.\n For example for " .
+ "--scsi0 local-lvm:0,discard=on --scsi1 local:0,cache=unsafe " .
+ "there must be --diskimages scsi0=/source/path,scsi1=/other/path";
+ my $device_count = grep { $use_import->($_) } keys %$param;
+
+ my $diskimages_count = @diskimage_pairs;
+ if ($device_count != $diskimages_count) {
+ raise_param_exc({diskimages => $msg});
+ }
+
+ my $diskimages = {};
+ foreach ( @diskimage_pairs ) {
+ my ($device, $diskimage) = split('=', $_);
+ $diskimages->{$device} = $diskimage;
+ }
+
+ my $worker = sub {
+ eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') };
+ die "Unable to create config for VM import: $@" if $@;
+
+ my @volids_of_imported = ();
+ eval { foreach my $opt (keys %$param) {
+ next if ($opt eq 'start');
+
+ my $updated_value;
+ if ($use_import->($opt)) {
+ # $opt is bus/device like ide0, scsi5
+
+ my $device = PVE::QemuServer::parse_drive($opt, $param->{$opt});
+ raise_param_exc({ $opt => "Unable to parse drive options" })
+ if !$device;
+
+ my $source_path = $convert_to_path->($diskimages->{$opt});
+
+ $param->{$opt} =~ m/format=([^,]*)/;
+ my $format = $1;
+
+ my $imported_volid = $import_disk_image->({
+ vmid => $vmid,
+ source => $source_path,
+ device => $opt,
+ storage => (split ':', $device->{file})[0],
+ format => $format,
+ });
+ push @volids_of_imported, $imported_volid;
+
+ # $param->{opt} has all required options but also dummy
+ # import 0 instead of the image
+ # for example, local-lvm:0,discard=on,mbps_rd=100
+ my $volid = $param->{$opt};
+ # Replace 0 with allocated volid, for example
+ # local-lvm:vm-100-disk-2,discard=on,mbps_rd=100
+ $volid =~ s/^.*?,/$imported_volid,/;
+
+ $updated_value = $volid;
+ } else {
+ $updated_value = $param->{$opt};
+ }
+ $update_vm_api->(
+ {
+ node => $node,
+ vmid => $vmid,
+ $opt => $updated_value,
+ skiplock => 1,
+ },
+ 1, # avoid nested workers that only do a short operation
+ );
+ }};
+
+ my $conf = PVE::QemuConfig->load_config($vmid);
+ my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf);
+ $update_vm_api->(
+ {
+ node => $node,
+ vmid => $vmid,
+ boot => PVE::QemuServer::print_bootorder($bootdevs),
+ skiplock => 1,
+ },
+ 1,
+ );
+
+ my $err = $@;
+ if ($err) {
+ foreach my $volid (@volids_of_imported) {
+ eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+ warn $@ if $@;
+ }
+
+ eval {
+ my $conffile = PVE::QemuConfig->config_file($vmid);
+ unlink($conffile) or die "Failed to remove config file: $!\n";
+ };
+ warn $@ if $@;
+
+ die $err;
+ }
+
+ eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
+ warn $@ if $@;
+
+ if ($param->{start}) {
+ PVE::QemuServer::vm_start($storecfg, $vmid);
+ }
+ };
+
+ return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker);
+ }});
+
+
1;
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9c65d76..c02f5eb 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -300,7 +300,7 @@ my $confdesc = {
optional => 1,
type => 'string',
description => "Lock/unlock the VM.",
- enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
+ enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended import)],
},
cpulimit => {
optional => 1,
@@ -998,6 +998,18 @@ sub verify_volume_id_or_qm_path {
return $volid;
}
+PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
+sub verify_volume_id_or_absolute_path {
+ my ($volid, $noerr) = @_;
+
+ # Exactly these 2 are allowed in id_or_qm_path but should not be allowed here
+ if ($volid eq 'none' || $volid eq 'cdrom') {
+ return undef if $noerr;
+ die "Invalid format! Should be volume ID or absolute path.";
+ }
+ return verify_volume_id_or_qm_path($volid, $noerr);
+}
+
my $usb_fmt = {
host => {
default_key => 1,
@@ -6659,7 +6671,7 @@ sub qemu_img_convert {
$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
$src_is_iscsi = ($src_path =~ m|^iscsi://|);
$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
- } elsif (-f $src_volid) {
+ } elsif (-f $src_volid || -b _) { # -b required to import from LVM images
$src_path = $src_volid;
if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
$src_format = $1;
diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm
index c76c199..36b7fff 100644
--- a/PVE/QemuServer/OVF.pm
+++ b/PVE/QemuServer/OVF.pm
@@ -87,7 +87,7 @@ sub id_to_pve {
# returns two references, $qm which holds qm.conf style key/values, and \@disks
sub parse_ovf {
- my ($ovf, $debug) = @_;
+ my ($ovf, $debug, $ignore_size) = @_;
my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1);
@@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id);
die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
}
- my $virtual_size;
- if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
- die "error parsing $backing_file_path, size seems to be $virtual_size\n";
+ my $virtual_size = 0;
+ if (!$ignore_size) { # Not possible if manifest is uploaded in web gui
+ if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
+ die "error parsing $backing_file_path: Could not get file size info: $@\n";
+ }
}
$pve_disk = {
--
2.20.1
More information about the pve-devel
mailing list