[pve-devel] [PATCH v6 qemu-server] Add API for import wizards
Dominic Jäger
d.jaeger at proxmox.com
Tue Mar 9 11:43:17 CET 2021
Extend qm importdisk/importovf functionality to the API.
Signed-off-by: Dominic Jäger <d.jaeger at proxmox.com>
---
v5->v6:
More parsing
Fix regex
Improve --boot handling
Move readovf from manager to qemu-server (like CPU)
Create properties helper for readovf return values
PVE/API2/Qemu.pm | 458 ++++++++++++++++++++++++++++++++++++++++-
PVE/API2/Qemu/Makefile | 2 +-
PVE/API2/Qemu/OVF.pm | 68 ++++++
PVE/QemuServer.pm | 32 ++-
PVE/QemuServer/OVF.pm | 10 +-
5 files changed, 562 insertions(+), 8 deletions(-)
create mode 100644 PVE/API2/Qemu/OVF.pm
diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index 6706b55..a689c9e 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -45,7 +45,6 @@ BEGIN {
}
}
-use Data::Dumper; # fixme: remove
use base qw(PVE::RESTHandler);
@@ -4383,4 +4382,461 @@ __PACKAGE__->register_method({
return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
}});
+# Raise exception if $format is not supported by $storeid
+my $check_format_is_supported = sub {
+ my ($format, $storeid, $storecfg) = @_;
+ die "You have to provide storage ID" if !$storeid;
+ die "You have to provide the storage configurration" if !$storecfg;
+
+ return if !$format;
+
+ my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+ my $supported = grep { $_ eq $format } @$valid_formats;
+
+ die "$format is not supported on storage $storeid" if !$supported;
+};
+
+# storecfg ... PVE::Storage::config()
+# vmid ... target VM ID
+# vmconf ... target VM configuration
+# source ... source image (volid or absolute path)
+# target ... hash with
+# storeid => storage ID
+# format => disk format (optional)
+# options => string with device options (may or may not contain <storeid>:0)
+# device => device where the disk is attached (for example, scsi3) (optional)
+#
+# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2)
+my $import_disk_image = sub {
+ my ($param) = @_;
+ my $storecfg = $param->{storecfg};
+ my $vmid = $param->{vmid};
+ my $vmconf = $param->{vmconf};
+ my $target = $param->{target};
+ my $requested_format = $target->{format};
+ my $storeid = $target->{storeid};
+
+ die "Source parameter is undefined!" if !defined $param->{source};
+ my $source = PVE::Storage::abs_filesystem_path($storecfg, $param->{source}, 1);
+
+ eval { PVE::Storage::storage_config($storecfg, $storeid) };
+ die "Error while importing disk image $source: $@\n" if $@;
+
+ my $src_size = PVE::Storage::file_size_info($source);
+ # Previous abs_filesystem_path performs additional checks
+ die "Could not get file size of $source" if !defined($src_size);
+
+ $check_format_is_supported->($requested_format, $storeid, $storecfg);
+
+ my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
+ $storecfg, $storeid, undef, $requested_format);
+ my $dst_volid = PVE::Storage::vdisk_alloc($storecfg, $storeid,
+ $vmid, $dst_format, undef, $src_size / 1024);
+
+ print "Importing disk image '$source'...\n";
+ eval {
+ local $SIG{INT} =
+ local $SIG{TERM} =
+ local $SIG{QUIT} =
+ local $SIG{HUP} =
+ local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
+
+ my $zeroinit = PVE::Storage::volume_has_feature($storecfg,
+ 'sparseinit', $dst_volid);
+
+ PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
+ PVE::QemuServer::qemu_img_convert($source, $dst_volid,
+ $src_size, undef, $zeroinit);
+ PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
+
+ };
+ if (my $err = $@) {
+ eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
+ warn "Cleanup of $dst_volid failed: $@ \n" if $@;
+
+ die "Importing disk '$source' failed: $err\n" if $err;
+ }
+
+ my $drive = $dst_volid;
+ if ($target->{device}) {
+ # Attach to target device with options if they are specified
+ if (defined $target->{options}) {
+ # Options string with or without storeid is allowed
+ # => Avoid potential duplicate storeid for update
+ $target->{options} =~ s/$storeid:0,?//; # ? for if only storeid:0 present
+ $drive .= ",$target->{options}" ;
+ }
+ } else {
+ $target->{device} = PVE::QemuConfig->add_unused_volume($vmconf, $dst_volid);
+ }
+ print "Imported '$source' to $dst_volid\n";
+ $update_vm_api->(
+ {
+ node => $target->{node},
+ vmid => $vmid,
+ $target->{device} => $drive,
+ skiplock => 1,
+ },
+ 1,
+ );
+
+ return $dst_volid;
+};
+
+__PACKAGE__->register_method ({
+ name => 'importdisk',
+ path => '{vmid}/importdisk',
+ method => 'POST',
+ proxyto => 'node',
+ protected => 1,
+ description => "Import an external disk image into a VM. The image format ".
+ "has to be supported by qemu-img.",
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid',
+ {completion => \&PVE::QemuServer::complete_vmid}),
+ source => {
+ description => "Disk image to import. Can be a volid ".
+ "(local:99/imageToImport.raw) or an absolute path on the server.",
+ type => 'string',
+ },
+ device => {
+ type => 'string',
+ description => "Bus/Device type of the new disk (e.g. 'ide0', ".
+ "'scsi2'). Will add the image as unused disk if omitted.",
+ enum => [PVE::QemuServer::Drive::valid_drive_names()],
+ optional => 1,
+ },
+ device_options => {
+ type => 'string',
+ description => "Options to set for the new disk (e.g. 'discard=on,backup=0')",
+ optional => 1,
+ requires => 'device',
+ },
+ storage => get_standard_option('pve-storage-id', {
+ description => "The storage to which the image will be imported to.",
+ completion => \&PVE::QemuServer::complete_storage,
+ }),
+ format => {
+ type => 'string',
+ description => 'Target format.',
+ enum => [ 'raw', 'qcow2', 'vmdk' ],
+ optional => 1,
+ },
+ digest => get_standard_option('pve-config-digest'),
+ },
+ },
+ returns => { type => 'string'},
+ code => sub {
+ my ($param) = @_;
+ my $vmid = extract_param($param, 'vmid');
+ my $node = extract_param($param, 'node');
+ my $source = extract_param($param, 'source');
+ my $digest_param = extract_param($param, 'digest');
+ my $device_options = extract_param($param, 'device_options');
+ my $device = extract_param($param, 'device');
+ my $storeid = extract_param($param, 'storage');
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+ my $storecfg = PVE::Storage::config();
+ PVE::Storage::storage_config($storecfg, $storeid); # check for errors
+
+ # Format can be set explicitly "--format vmdk"
+ # or as part of device options "--device_options discard=on,format=vmdk"
+ # or not at all, but not both together
+ my $device_options_format;
+ if ($device_options) {
+ # parse device_options string according to disk schema for
+ # validation and to make usage easier
+
+ # any existing storage ID is OK to get a valid (fake) string for parse_drive
+ my $valid_string = "$storeid:0,$device_options";
+
+ # member "file" is fake
+ my $drive_full = PVE::QemuServer::Drive::parse_drive($device, $valid_string);
+ $device_options_format = $drive_full->{format};
+ }
+
+ my $format = extract_param($param, 'format'); # may be undefined
+ if ($device_options) {
+ if ($format && $device_options_format) {
+ raise_param_exc({format => "Format already specified in device_options!"});
+ } else {
+ $format = $format || $device_options_format; # may still be undefined
+ }
+ }
+
+ $check_format_is_supported->($format, $storeid, $storecfg);
+
+ # provide a useful error (in the API response) before forking
+ my $no_lock_conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->check_lock($no_lock_conf);
+ PVE::Tools::assert_if_modified($no_lock_conf->{digest}, $digest_param);
+ if ($device && $no_lock_conf->{$device}) {
+ die "Could not import because device $device is already in ".
+ "use in VM $vmid. Choose a different device!";
+ }
+
+ my $worker = sub {
+ my $conf;
+ PVE::QemuConfig->lock_config($vmid, sub {
+ $conf = PVE::QemuConfig->load_config($vmid);
+ PVE::QemuConfig->check_lock($conf);
+
+ # Our device-in-use check may be invalid if the new conf is different
+ PVE::Tools::assert_if_modified($conf->{digest}, $no_lock_conf->{digest});
+
+ PVE::QemuConfig->set_lock($vmid, 'import');
+ });
+
+ my $target = {
+ node => $node,
+ storeid => $storeid,
+ };
+ # Avoid keys with undef values
+ $target->{format} = $format if defined $format;
+ $target->{device} = $device if defined $device;
+ $target->{options} = $device_options if defined $device_options;
+ $import_disk_image->({
+ storecfg => $storecfg,
+ vmid => $vmid,
+ vmconf => $conf,
+ source => $source,
+ target => $target,
+ });
+
+ PVE::QemuConfig->remove_lock($vmid, 'import');
+ };
+ return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker);
+ }});
+
+__PACKAGE__->register_method({
+ name => 'importvm',
+ path => '{vmid}/importvm',
+ method => 'POST',
+ description => "Import a VM from existing disk images.",
+ protected => 1,
+ proxyto => 'node',
+ parameters => {
+ additionalProperties => 0,
+ properties => PVE::QemuServer::json_config_properties(
+ {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid', { completion =>
+ \&PVE::Cluster::complete_next_vmid }),
+ diskimages => {
+ description => "Mapping of devices to disk images. For " .
+ "example, scsi0=/mnt/nfs/image1.vmdk,scsi1=/mnt/nfs/image2",
+ type => 'string',
+ },
+ start => {
+ optional => 1,
+ type => 'boolean',
+ default => 0,
+ description => "Start VM after it was imported successfully.",
+ },
+ }),
+ },
+ returns => {
+ type => 'string',
+ },
+ code => sub {
+ my ($param) = @_;
+ my $node = extract_param($param, 'node');
+ my $vmid = extract_param($param, 'vmid');
+ my $diskimages_string = extract_param($param, 'diskimages');
+ my $boot = extract_param($param, 'boot');
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+ my $storecfg = PVE::Storage::config();
+
+ PVE::Cluster::check_cfs_quorum();
+
+ # Return true iff $opt is to be imported, that means it is a device
+ # like scsi2 and the special import syntax/zero is specified for it,
+ # for example local-lvm:0 but not local-lvm:5
+ my $is_import = sub {
+ my ($opt) = @_;
+ return 0 if $opt eq 'efidisk0';
+ if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
+ my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
+ return $drive->{file} =~ m/0$/;
+ }
+ return 0;
+ };
+
+ # bus/device like ide0, scsi5 where the imported disk images get attached
+ my $target_devices = [];
+ # List of VM parameters like memory, cpu type, but also disks that are newly created
+ my $vm_params = [];
+ foreach my $opt (keys %$param) {
+ next if ($opt eq 'start'); # does not belong in config
+ # New function, so we can forbid deprecated
+ raise_param_exc({bootdisk => "Deprecated: Use --boot order= instead"})
+ if $opt eq 'bootdisk';
+
+ my $list = $is_import->($opt) ? $target_devices : $vm_params;
+ push @$list, $opt;
+ }
+
+ my $diskimages = {};
+ foreach (split ',', $diskimages_string) {
+ my ($device, $diskimage) = split('=', $_);
+ $diskimages->{$device} = $diskimage;
+ if (defined $param->{$device}) {
+ my $drive = PVE::QemuServer::parse_drive($device, $param->{$device});
+ $drive->{file} =~ m/(\d)$/;
+ if ($1 != 0) {
+ raise_param_exc({
+ $device => "Each entry of --diskimages must have a ".
+ "corresponding device with special import syntax " .
+ "(e.g. --scsi3 local-lvm:0). $device from " .
+ "--diskimages has a matching device but the size " .
+ "of that is $1 instead of 0!",
+ });
+ }
+ } else {
+ # It is possible to also create new empty disk images during
+ # import by adding something like scsi2=local:10, therefore
+ # vice-versa check is not required
+ raise_param_exc({
+ diskimages => "There must be a matching device for each " .
+ "--diskimages entry that should be imported, but " .
+ "there is no matching device for $device!\n" .
+ " For example, for --diskimages scsi0=/source/path,scsi1=/other/path " .
+ "there must be --scsi0 local-lvm:0,discard=on --scsi1 local:0,cache=unsafe",
+ });
+ }
+ # Dies if $diskimage cannot be found
+ PVE::Storage::abs_filesystem_path($storecfg, $diskimage, 1);
+ }
+ foreach my $device (@$target_devices) {
+ my $drive = PVE::QemuServer::parse_drive($device, $param->{$device});
+ if ($drive->{file} =~ m/0$/) {
+ if (!defined $diskimages->{$device}) {
+ raise_param_exc({
+ $device => "Each device with the special import " .
+ "syntax (the 0) must have a corresponding in " .
+ "--diskimages that specifies the source of the " .
+ "import, but there is no such entry for $device!",
+ });
+ }
+ }
+ }
+
+ # After devices are ensured to be correct
+ if ($boot) {
+ my $new_bootcfg = PVE::JSONSchema::parse_property_string('pve-qm-boot', $boot);
+ if ($new_bootcfg->{order}) {
+ my @devs = PVE::Tools::split_list($new_bootcfg->{order});
+ for my $dev (@devs) {
+ my $will_be_imported = grep (/^$dev$/, @$target_devices);
+ my $will_be_created = grep (/^$dev$/, @$vm_params);
+ if ( !($will_be_imported || $will_be_created)) {
+ raise_param_exc({boot => "$dev will be neither imported " .
+ "nor created, so it cannot be a boot device!"});
+ }
+ }
+ } else {
+ raise_param_exc({boot => "Deprecated: Use --boot order= instead"});
+ }
+ }
+
+ eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') };
+ die "Unable to create config for VM import: $@" if $@;
+
+ my $worker = sub {
+ my $get_conf = sub {
+ my ($vmid) = @_;
+ PVE::QemuConfig->lock_config($vmid, sub {
+ my $conf = PVE::QemuConfig->load_config($vmid);
+ if (PVE::QemuConfig->has_lock($conf, 'import')) {
+ return $conf;
+ } else {
+ die "import lock in VM $vmid config file missing!";
+ }
+ });
+ };
+
+ $get_conf->($vmid); # quick check for lock
+
+ my $short_update = {
+ node => $node,
+ vmid => $vmid,
+ skiplock => 1,
+ };
+ foreach ( @$vm_params ) {
+ $short_update->{$_} = $param->{$_};
+ }
+ $update_vm_api->($short_update, 1); # writes directly in config file
+
+ my $conf = $get_conf->($vmid);
+
+ # When all short updates were succesfull, then the long imports
+ my @imported_successfully = ();
+ eval { foreach my $device (@$target_devices) {
+ my $param_parsed = PVE::QemuServer::parse_drive($device, $param->{$device});
+ die "Parsing $param->{$device} failed" if !$param_parsed;
+ my $storeid = PVE::Storage::parse_volume_id($param_parsed->{file});
+
+ my $imported = $import_disk_image->({
+ storecfg => $storecfg,
+ vmid => $vmid,
+ vmconf => $conf,
+ source => $diskimages->{$device},
+ target => {
+ storeid => $storeid,
+ format => $param_parsed->{format},
+ options => $param->{$device},
+ device => $device,
+ },
+ });
+ push @imported_successfully, $imported;
+ }};
+ my $err = $@;
+ if ($err) {
+ foreach my $volid (@imported_successfully) {
+ eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+ warn $@ if $@;
+ }
+
+ eval {
+ my $conffile = PVE::QemuConfig->config_file($vmid);
+ unlink($conffile) or die "Failed to remove config file: $!";
+ };
+ warn $@ if $@;
+
+ die "Import aborted: $err";
+ }
+ if (!$boot) {
+ $conf = $get_conf->($vmid); # import_disk_image changed config file directly
+ my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf);
+ $boot = PVE::QemuServer::print_bootorder($bootdevs);
+ }
+ $update_vm_api->(
+ {
+ node => $node,
+ vmid => $vmid,
+ boot => $boot,
+ skiplock => 1,
+ },
+ 1,
+ );
+
+ eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
+ warn $@ if $@;
+
+ if ($param->{start}) {
+ PVE::QemuServer::vm_start($storecfg, $vmid);
+ }
+ };
+
+ return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker);
+ }});
+
+
1;
diff --git a/PVE/API2/Qemu/Makefile b/PVE/API2/Qemu/Makefile
index 5d4abda..bdd4762 100644
--- a/PVE/API2/Qemu/Makefile
+++ b/PVE/API2/Qemu/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Agent.pm CPU.pm Machine.pm
+SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm
.PHONY: install
install:
diff --git a/PVE/API2/Qemu/OVF.pm b/PVE/API2/Qemu/OVF.pm
new file mode 100644
index 0000000..bd6e90b
--- /dev/null
+++ b/PVE/API2/Qemu/OVF.pm
@@ -0,0 +1,68 @@
+package PVE::API2::Qemu::OVF;
+
+use strict;
+use warnings;
+
+use PVE::RESTHandler;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::QemuServer::OVF;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'index',
+ path => '',
+ method => 'GET',
+ proxyto => 'node',
+ description => "Read an .ovf manifest.",
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ manifest => {
+ description => ".ovf manifest",
+ type => 'string',
+ },
+ },
+ },
+ returns => {
+ description => "VM config according to .ovf manifest and digest of manifest",
+ type => "object",
+ },
+ returns => {
+ type => 'object',
+ additionalProperties => 1,
+ properties => PVE::QemuServer::json_ovf_properties({
+ name => {
+ type => 'string',
+ optional => 1,
+ },
+ cores => {
+ type => 'integer',
+ optional => 1,
+ },
+ memory => {
+ type => 'integer',
+ optional => 1,
+ },
+ }),
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $manifest = $param->{manifest};
+ die "$manifest: non-existent or non-regular file\n" if (! -f $manifest);
+
+ my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest, 0, 1);
+ my $result;
+ $result->{cores} = $parsed->{qm}->{cores};
+ $result->{name} = $parsed->{qm}->{name};
+ $result->{memory} = $parsed->{qm}->{memory};
+ my $disks = $parsed->{disks};
+ foreach my $disk (@$disks) {
+ $result->{$disk->{disk_address}} = $disk->{backing_file};
+ }
+ return $result;
+ }});
+
+1;
\ No newline at end of file
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 1410ecb..d4017b7 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -300,7 +300,7 @@ my $confdesc = {
optional => 1,
type => 'string',
description => "Lock/unlock the VM.",
- enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
+ enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended import)],
},
cpulimit => {
optional => 1,
@@ -998,6 +998,18 @@ sub verify_volume_id_or_qm_path {
return $volid;
}
+PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
+sub verify_volume_id_or_absolute_path {
+ my ($volid, $noerr) = @_;
+
+ # Exactly these 2 are allowed in id_or_qm_path but should not be allowed here
+ if ($volid eq 'none' || $volid eq 'cdrom') {
+ return undef if $noerr;
+ die "Invalid format! Should be volume ID or absolute path.";
+ }
+ return verify_volume_id_or_qm_path($volid, $noerr);
+}
+
my $usb_fmt = {
host => {
default_key => 1,
@@ -2030,6 +2042,22 @@ sub json_config_properties {
return $prop;
}
+# Properties that we can read from an OVF file
+sub json_ovf_properties {
+ my $prop = shift;
+
+ foreach my $device ( PVE::QemuServer::Drive::valid_drive_names()) {
+ $prop->{$device} = {
+ type => 'string',
+ format => 'pve-volume-id-or-qm-path',
+ description => "Disk image that gets imported to $device",
+ optional => 1,
+ };
+ }
+
+ return $prop;
+}
+
# return copy of $confdesc_cloudinit to generate documentation
sub cloudinit_config_properties {
@@ -6722,7 +6750,7 @@ sub qemu_img_convert {
$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
$src_is_iscsi = ($src_path =~ m|^iscsi://|);
$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
- } elsif (-f $src_volid) {
+ } elsif (-f $src_volid || -b _) { # -b required to import from LVM images
$src_path = $src_volid;
if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
$src_format = $1;
diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm
index c76c199..36b7fff 100644
--- a/PVE/QemuServer/OVF.pm
+++ b/PVE/QemuServer/OVF.pm
@@ -87,7 +87,7 @@ sub id_to_pve {
# returns two references, $qm which holds qm.conf style key/values, and \@disks
sub parse_ovf {
- my ($ovf, $debug) = @_;
+ my ($ovf, $debug, $ignore_size) = @_;
my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1);
@@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id);
die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
}
- my $virtual_size;
- if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
- die "error parsing $backing_file_path, size seems to be $virtual_size\n";
+ my $virtual_size = 0;
+ if (!$ignore_size) { # Not possible if manifest is uploaded in web gui
+ if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
+ die "error parsing $backing_file_path: Could not get file size info: $@\n";
+ }
}
$pve_disk = {
--
2.20.1
More information about the pve-devel
mailing list