[pve-devel] [PATCH v7 qemu-server] Add API for import wizards
Fabian Grünbichler
f.gruenbichler at proxmox.com
Wed Mar 31 17:12:28 CEST 2021
this is starting to shape up nicely. as promised, I now took a stab at
(roughly!) integrating this into our regular flow (see diff below):
- IMPORT_DISK_RE now uses -1, as 0 actually can be mismatched by
NEW_DISK_RE
- the actual import happens in create_disks
- only the basic checks (match of "marked for import" with "import
sources") happen early on
- the target storage and source volume are permission checked now
- importvm is dropped in favor of calling create_vm with import_sources
- update_vm_async now also supports import_sources
the last two IMHO have some nice benefits:
- we can now easily mix and match importing and newly allocating blank
disks in a single create call (allowing us to use a single GUI wizard
as well if we want)
- create_vm and importvm don't have to duplicate all the surrounding
logic (or have strange contortions to make one call the other)
- we could likely drop the separate import_disk API call, and let the
`importdisk` CLI command prepare parameters for a regular VM config
update
I'm sure I've missed some corner cases, as I've only tested create_vm
with importing, and not the other newly exposed APIs.
stat shows how much boilerplace/duplication this removes, although there
is probably even more potential here since `create_vm` partly duplicates
the `update_vm_api` sub that ends up calling `create_disks`:
PVE/API2/Qemu.pm | 481 ++++++++++++++++++++-----------------------------------
1 file changed, 171 insertions(+), 310 deletions(-)
the original patch was +381 for that file, so in total we are now at
+242 instead.
could you take a look and see if I missed anything fundamental?
-----8<-----
diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index 2f50f38..41e1ab7 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -60,8 +60,19 @@ my $resolve_cdrom_alias = sub {
}
};
+my $parse_import_sources = sub {
+ my $param = shift;
+ my $import = {};
+ foreach my $pair (PVE::Tools::split_list($param)) {
+ my ($device, $diskimage) = split('=', $pair);
+ $import->{$device} = $diskimage;
+ }
+
+ return $import;
+};
+
my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
-my $IMPORT_DISK_RE = qr!^([^/:\s]+):0$!;
+my $IMPORT_DISK_RE = qr!^([^/:\s]+):-1$!;
my $check_storage_access = sub {
my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
@@ -84,6 +95,12 @@ my $check_storage_access = sub {
my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
if !$scfg->{content}->{images};
+ } elsif ($volid =~ $IMPORT_DISK_RE) {
+ my $storeid = $1;
+ $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
+ my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+ raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
+ if !$scfg->{content}->{images};
} else {
PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
}
@@ -91,6 +108,13 @@ my $check_storage_access = sub {
$rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
if defined($settings->{vmstatestorage});
+
+ if (defined($settings->{import_sources})) {
+ my $images = $parse_import_sources->($settings->{import_sources});
+ foreach my $source_image (values %$images) {
+ PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $source_image);
+ }
+ }
};
my $check_storage_access_clone = sub {
@@ -133,10 +157,25 @@ my $check_storage_access_clone = sub {
return $sharedvm;
};
+# Raise exception if $format is not supported by $storeid
+my $check_format_is_supported = sub {
+ my ($format, $storeid, $storecfg) = @_;
+ die "storage ID parameter must be passed to the sub" if !$storeid;
+ die "storage configuration must be passed to the sub" if !$storecfg;
+
+ return if !$format;
+
+ my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+ my $supported = grep { $_ eq $format } @$valid_formats;
+
+ die "format '$format' is not supported on storage $storeid" if !$supported;
+};
+
+
# Note: $pool is only needed when creating a VM, because pool permissions
# are automatically inherited if VM already exists inside a pool.
my $create_disks = sub {
- my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
+ my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage, $import) = @_;
my $vollist = [];
@@ -192,6 +231,69 @@ my $create_disks = sub {
$disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
delete $disk->{format}; # no longer needed
$res->{$ds} = PVE::QemuServer::print_drive($disk);
+ } elsif ($volid =~ $IMPORT_DISK_RE) {
+ my $target_storage = $1;
+
+ my $source = $import->{$ds};
+ die "cannot import '$ds', no import source defined\n" if !$source;
+ $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
+ my $src_size = PVE::Storage::file_size_info($source);
+ die "Could not get file size of $source" if !defined($src_size);
+
+ $check_format_is_supported->($disk->{format}, $storeid, $storecfg);
+
+ my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
+ $storecfg,
+ $storeid,
+ undef,
+ $disk->{format},
+ );
+ my $dst_volid = PVE::Storage::vdisk_alloc(
+ $storecfg,
+ $storeid,
+ $vmid,
+ $dst_format,
+ undef,
+ PVE::Tools::convert_size($src_size, 'b' => 'kb'),
+ );
+
+ print "Importing disk image '$source' as '$dst_volid'...\n";
+ eval {
+ local $SIG{INT} =
+ local $SIG{TERM} =
+ local $SIG{QUIT} =
+ local $SIG{HUP} =
+ local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
+
+ my $zeroinit = PVE::Storage::volume_has_feature(
+ $storecfg,
+ 'sparseinit',
+ $dst_volid,
+ );
+ PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
+ PVE::QemuServer::qemu_img_convert(
+ $source,
+ $dst_volid,
+ $src_size,
+ undef,
+ $zeroinit,
+ );
+ PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
+
+ };
+ if (my $err = $@) {
+ eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
+ warn "Cleanup of $dst_volid failed: $@ \n" if $@;
+
+ die "Importing disk '$source' failed: $err\n" if $err;
+ }
+ push @$vollist, $dst_volid;
+ $disk->{file} = $dst_volid;
+ if ($ds !~ /^unused\d+$/) {
+ $disk->{size} = $src_size;
+ delete $disk->{format}; # no longer needed
+ }
+ $res->{$ds} = PVE::QemuServer::print_drive($disk);
} else {
PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
@@ -218,7 +320,7 @@ my $create_disks = sub {
}
};
- eval { PVE::QemuConfig->foreach_volume($settings, $code); };
+ eval { PVE::QemuConfig->foreach_volume_full($settings, { include_unused => 1 }, $code); };
# free allocated images on error
if (my $err = $@) {
@@ -544,6 +646,13 @@ __PACKAGE__->register_method({
default => 0,
description => "Start VM after it was created successfully.",
},
+ import_sources => {
+ description => "\\0 delimited mapping of devices to disk images to import." .
+ "For example, scsi0=/mnt/nfs/image1.vmdk",
+ type => 'string',
+ format => 'device-image-pair-alist',
+ optional => 1,
+ },
}),
},
returns => {
@@ -608,21 +717,34 @@ __PACKAGE__->register_method({
&$check_cpu_model_access($rpcenv, $authuser, $param);
+ my $import_devices = $parse_import_sources->($param->{import_sources});
+
foreach my $opt (keys %$param) {
if (PVE::QemuServer::is_valid_drivename($opt)) {
my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
+ raise_param_exc({ $opt => "not marked for import, but import source defined" })
+ if $drive->{file} !~ $IMPORT_DISK_RE && $import_devices->{$opt};
+ raise_param_exc({ $opt => "marked for import, but no import source defined" })
+ if $drive->{file} =~ $IMPORT_DISK_RE && !$import_devices->{$opt};
PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
$param->{$opt} = PVE::QemuServer::print_drive($drive);
}
}
+ foreach my $opt (keys %$import_devices) {
+ raise_param_exc({ import_sources => "$opt not marked for import, but import source defined" })
+ if !defined($param->{$opt});
+
+ }
PVE::QemuServer::add_random_macs($param);
} else {
my $keystr = join(' ', keys %$param);
raise_param_exc({ archive => "option conflicts with other options ($keystr)"}) if $keystr;
+ raise_param_exc({ import_sources => "cannot import existing disk and restore backup." }) if $param->{import_sources};
+
if ($archive eq '-') {
die "pipe requires cli environment\n"
if $rpcenv->{type} ne 'cli';
@@ -690,10 +812,11 @@ __PACKAGE__->register_method({
my $realcmd = sub {
my $conf = $param;
my $arch = PVE::QemuServer::get_vm_arch($conf);
+ my $import = $parse_import_sources->(extract_param($param, "import_sources"));
my $vollist = [];
eval {
- $vollist = &$create_disks($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage);
+ $vollist = &$create_disks($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage, $import);
if (!$conf->{boot}) {
my $devs = PVE::QemuServer::get_default_bootdevices($conf);
@@ -1163,11 +1286,17 @@ my $update_vm_api = sub {
die "cannot add non-replicatable volume to a replicated VM\n";
};
+ my $import_devices = $parse_import_sources->($param->{import_sources});
+
foreach my $opt (keys %$param) {
if (PVE::QemuServer::is_valid_drivename($opt)) {
# cleanup drive path
my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
+ raise_param_exc({ $opt => "not marked for import, but import source defined" })
+ if $drive->{file} !~ $IMPORT_DISK_RE && $import_devices->{$opt};
+ raise_param_exc({ $opt => "marked for import, but no import source defined" })
+ if $drive->{file} =~ $IMPORT_DISK_RE && !$import_devices->{$opt};
PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
$check_replication->($drive);
$param->{$opt} = PVE::QemuServer::print_drive($drive);
@@ -1185,12 +1314,20 @@ my $update_vm_api = sub {
}
}
+ foreach my $opt (keys %$import_devices) {
+ raise_param_exc({ import_sources => "$opt not marked for import, but import source defined" })
+ if !defined($param->{$opt});
+
+ }
+
&$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
&$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
&$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
+ delete $param->{import_sources};
+
my $updatefn = sub {
my $conf = PVE::QemuConfig->load_config($vmid);
@@ -1332,7 +1469,7 @@ my $update_vm_api = sub {
PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
if defined($conf->{pending}->{$opt});
- &$create_disks($rpcenv, $authuser, $conf->{pending}, $arch, $storecfg, $vmid, undef, {$opt => $param->{$opt}});
+ &$create_disks($rpcenv, $authuser, $conf->{pending}, $arch, $storecfg, $vmid, undef, {$opt => $param->{$opt}}, {$opt => $import_devices->{$opt}});
} elsif ($opt =~ m/^serial\d+/) {
if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
$rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
@@ -1476,6 +1613,13 @@ __PACKAGE__->register_method({
optional => 1,
requires => 'delete',
},
+ import_sources => {
+ description => "\\0 delimited mapping of devices to disk images to import." .
+ "For example, scsi0=/mnt/nfs/image1.vmdk",
+ type => 'string',
+ format => 'device-image-pair-alist',
+ optional => 1,
+ },
digest => {
type => 'string',
description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
@@ -4377,111 +4521,6 @@ __PACKAGE__->register_method({
return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
}});
-# Raise exception if $format is not supported by $storeid
-my $check_format_is_supported = sub {
- my ($format, $storeid, $storecfg) = @_;
- die "storage ID parameter must be passed to the sub" if !$storeid;
- die "storage configuration must be passed to the sub" if !$storecfg;
-
- return if !$format;
-
- my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
- my $supported = grep { $_ eq $format } @$valid_formats;
-
- die "format '$format' is not supported on storage $storeid" if !$supported;
-};
-
-# storecfg ... PVE::Storage::config()
-# vmid ... target VM ID
-# vmconf ... target VM configuration
-# source ... source image (volid or absolute path)
-# target ... hash with
-# storeid => storage ID
-# format => disk format (optional)
-# options => hash with device options (may or may not contain <storeid>:0)
-# device => device where the disk is attached (for example, scsi3) (optional)
-#
-# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2)
-my $import_disk_image = sub {
- my ($storecfg, $vmid, $vmconf, $source, $target) = @_;
- my $requested_format = $target->{format};
- my $storeid = $target->{storeid};
-
- die "Source parameter is undefined!" if !defined $source;
- $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
-
- eval { PVE::Storage::storage_config($storecfg, $storeid) };
- die "Error while importing disk image $source: $@\n" if $@;
-
- my $src_size = PVE::Storage::file_size_info($source);
- die "Could not get file size of $source" if !defined($src_size);
-
- $check_format_is_supported->($requested_format, $storeid, $storecfg);
-
- my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
- $storecfg,
- $storeid,
- undef,
- $requested_format,
- );
- my $dst_volid = PVE::Storage::vdisk_alloc(
- $storecfg,
- $storeid,
- $vmid,
- $dst_format,
- undef,
- $src_size / 102,
- );
-
- print "Importing disk image '$source' as '$dst_volid'...\n";
- eval {
- local $SIG{INT} =
- local $SIG{TERM} =
- local $SIG{QUIT} =
- local $SIG{HUP} =
- local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
-
- my $zeroinit = PVE::Storage::volume_has_feature(
- $storecfg,
- 'sparseinit',
- $dst_volid,
- );
- PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
- PVE::QemuServer::qemu_img_convert(
- $source,
- $dst_volid,
- $src_size,
- undef,
- $zeroinit,
- );
- PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
-
- };
- if (my $err = $@) {
- eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
- warn "Cleanup of $dst_volid failed: $@ \n" if $@;
-
- die "Importing disk '$source' failed: $err\n" if $err;
- }
-
- $target->{options}->{file} = $dst_volid;
- my $options_string = PVE::QemuServer::print_drive($target->{options});
- $target->{device} = PVE::QemuConfig->add_unused_volume($vmconf, $dst_volid)
- if !$target->{device};
-
- $update_vm_api->(
- {
- vmid => $vmid,
- $target->{device} => $options_string,
- skiplock => 1,
- digest => $vmconf->{digest},
- },
- 1,
- );
-
- return $dst_volid;
-};
-
__PACKAGE__->register_method ({
name => 'importdisk',
path => '{vmid}/importdisk',
@@ -4544,22 +4583,25 @@ __PACKAGE__->register_method ({
PVE::Storage::storage_config($storecfg, $storeid);
- if ($device_options) {
- # $device_options may or may not contain <storeid>:0
- my $parsed = PVE::QemuServer::Drive::parse_drive($device, $device_options);
- if ($parsed) {
- raise_param_exc({$device_options => "Invalid import syntax"})
- if !($parsed->{file} =~ $IMPORT_DISK_RE);
- } else {
- my $fake = "$storeid:0,$device_options";
- $parsed = PVE::QemuServer::Drive::parse_drive($device, $fake);
- }
- delete $parsed->{file};
- delete $parsed->{interface};
- delete $parsed->{index};
- $device_options = $parsed;
+ if (!$device_options) {
+ $device_options = "$storeid:0";
+ }
+
+ # $device_options may or may not contain <storeid>:0
+ my $parsed = PVE::QemuServer::Drive::parse_drive($device, $device_options);
+
+ if ($parsed) {
+ raise_param_exc({$device_options => "Invalid import syntax"})
+ if !($parsed->{file} =~ $IMPORT_DISK_RE);
+ } else {
+ my $fake = "$storeid:0,$device_options";
+ $parsed = PVE::QemuServer::Drive::parse_drive($device, $fake);
}
+ delete $parsed->{interface};
+ delete $parsed->{index};
+ $device_options = $parsed;
+
# Format can be set explicitly "--format vmdk"
# or as part of device options "--device_options discard=on,format=vmdk"
my $format = extract_param($param, 'format');
@@ -4570,192 +4612,11 @@ __PACKAGE__->register_method ({
}
$check_format_is_supported->($format, $storeid, $storecfg);
- # quick checks before fork + lock
- my $conf = PVE::QemuConfig->load_config($vmid);
- PVE::QemuConfig->check_lock($conf);
- PVE::Tools::assert_if_modified($conf->{digest}, $digest);
- if ($device && $conf->{$device}) {
- die "Could not import because device $device is already in ".
- "use in VM $vmid. Choose a different device!";
- }
-
- my $worker = sub {
- PVE::QemuConfig->lock_config($vmid, sub {
- $conf = PVE::QemuConfig->load_config($vmid);
- PVE::QemuConfig->check_lock($conf);
-
- PVE::Tools::assert_if_modified($conf->{digest}, $digest);
- PVE::QemuConfig->set_lock($vmid, 'import');
- $conf = PVE::QemuConfig->load_config($vmid);
- });
-
- my $target = {
- node => $node,
- storeid => $storeid,
- };
- $target->{format} = $format;
- $target->{device} = $device;
- $target->{options} = $device_options;
- eval { $import_disk_image->($storecfg, $vmid, $conf, $source, $target) };
- my $err = $@;
- eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
- warn $@ if $@;
- die $err if $err;
- };
- return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker);
+ return $update_vm_api->({
+ $device => PVE::QemuServer::Drive::print_drive($device_options),
+ import_sources => "$device=$source",
+ digest => $digest,
+ });
}});
-__PACKAGE__->register_method({
- name => 'importvm',
- path => '{vmid}/importvm',
- method => 'POST',
- description => "Import a VM from existing disk images.",
- protected => 1,
- proxyto => 'node',
- parameters => {
- additionalProperties => 0,
- properties => PVE::QemuServer::json_config_properties(
- {
- node => get_standard_option('pve-node'),
- vmid => get_standard_option('pve-vmid', { completion =>
- \&PVE::Cluster::complete_next_vmid }),
- diskimage => {
- description => "\\0 delimited mapping of devices to disk images. For " .
- "example, scsi0=/mnt/nfs/image1.vmdk",
- type => 'string',
- format => 'device-image-pair-alist',
- },
- start => {
- optional => 1,
- type => 'boolean',
- default => 0,
- description => "Start VM after it was imported successfully.",
- },
- }),
- },
- returns => {
- type => 'string',
- },
- code => sub {
- my ($param) = @_;
- my $node = extract_param($param, 'node');
- my $vmid = extract_param($param, 'vmid');
- my $diskimages_string = extract_param($param, 'diskimage');
- my $boot = extract_param($param, 'boot');
- my $start = extract_param($param, 'start');
-
- my $rpcenv = PVE::RPCEnvironment::get();
- my $authuser = $rpcenv->get_user();
- my $storecfg = PVE::Storage::config();
-
- PVE::Cluster::check_cfs_quorum();
-
- my $import_param = {};
- foreach my $opt (keys %$param) {
- next if $opt eq 'efidisk0';
- raise_param_exc({bootdisk => "Deprecated: Use --boot order= instead"})
- if $opt eq 'bootdisk';
-
- if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
- my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
- if ($drive->{file} =~ $IMPORT_DISK_RE) {
- $import_param->{$opt} = $drive;
- delete $param->{$opt};
- }
- }
- }
-
- my $diskimages = {};
- foreach my $pair (PVE::Tools::split_list($diskimages_string)) {
- my ($device, $diskimage) = split('=', $pair);
- $diskimages->{$device} = $diskimage;
- raise_param_exc({
- $device => "Device '$device' not marked for import, " .
- "but import source '$diskimage' specified",
- }) if !defined($import_param->{$device});
- PVE::Storage::abs_filesystem_path($storecfg, $diskimage, 1);
- }
-
- foreach my $device (keys %$import_param) {
- raise_param_exc({
- $device => "Device '$device' marked for import, but no source given\n",
- }) if !defined($diskimages->{$device});
- }
-
- eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') };
- die "Unable to create config for VM import: $@" if $@;
-
- my $worker = sub {
- my $reload_conf = sub {
- my ($vmid) = @_;
- my $conf = PVE::QemuConfig->load_config($vmid);
- return $conf if PVE::QemuConfig->has_lock($conf, 'import');
- die "import lock in VM $vmid config file missing!";
- };
-
- my $conf = $reload_conf->($vmid);
- $update_vm_api->(
- {
- %$param,
- node => $node,
- vmid => $vmid,
- skiplock => 1,
- digest => $conf->{digest},
- },
- 1
- );
-
- eval {
- foreach my $device (keys %$import_param) {
- $conf = $reload_conf->($vmid);
- my $drive = $import_param->{$device};
- my $storeid = PVE::Storage::parse_volume_id($drive->{file});
- my $imported = $import_disk_image->(
- $storecfg,
- $vmid,
- $conf,
- $diskimages->{$device},
- {
- storeid => $storeid,
- format => $drive->{format},
- options => $drive,
- device => $device,
- },
- );
- }
- };
- my $err = $@;
- if ($err) {
- eval { PVE::QemuServer::destroy_vm($storecfg, $vmid, 1) };
- warn "Could not destroy VM $vmid: $@" if "$@";
-
- die "Import failed: $err";
- }
-
- $conf = $reload_conf->($vmid);
- if (!$boot) {
- my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf);
- $boot = PVE::QemuServer::print_bootorder($bootdevs);
- }
- $update_vm_api->(
- {
- node => $node,
- vmid => $vmid,
- boot => $boot,
- skiplock => 1,
- digest => $conf->{digest},
- },
- 1,
- );
-
- eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
- warn $@ if $@;
-
- PVE::QemuServer::vm_start($storecfg, $vmid) if $start;
- };
-
- return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker);
- }});
-
-
1;
----->8-----
On March 26, 2021 1:32 pm, Dominic Jäger wrote:
> Extend qm importdisk/importovf functionality to the API.
>
> Signed-off-by: Dominic Jäger <d.jaeger at proxmox.com>
>
> ---
> v6->v7: Feedback by Fabian G
> - Introduce a regex for the import syntax <storeid>:0
> - Use parameter list instead of hash for import helper
> - More parsing, less string magic
> - More VM config digest checking
> - Create a schema format for diskimage source mapping
> - Preliminarily remove some boot parameter handling
> - Dare to really edit schema format subs for a cleaner solution
> - Whitespace, variable names, ...
>
> PVE/API2/Qemu.pm | 383 ++++++++++++++++++++++++++++++++++++++++-
> PVE/API2/Qemu/Makefile | 2 +-
> PVE/API2/Qemu/OVF.pm | 68 ++++++++
> PVE/QemuServer.pm | 52 +++++-
> PVE/QemuServer/OVF.pm | 10 +-
> 5 files changed, 502 insertions(+), 13 deletions(-)
> create mode 100644 PVE/API2/Qemu/OVF.pm
>
> diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
> index e95ab13..2f50f38 100644
> --- a/PVE/API2/Qemu.pm
> +++ b/PVE/API2/Qemu.pm
> @@ -45,7 +45,6 @@ BEGIN {
> }
> }
>
> -use Data::Dumper; # fixme: remove
>
> use base qw(PVE::RESTHandler);
>
> @@ -62,6 +61,7 @@ my $resolve_cdrom_alias = sub {
> };
>
> my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
> +my $IMPORT_DISK_RE = qr!^([^/:\s]+):0$!;
> my $check_storage_access = sub {
> my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
>
> @@ -4377,4 +4377,385 @@ __PACKAGE__->register_method({
> return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
> }});
>
> +# Raise exception if $format is not supported by $storeid
> +my $check_format_is_supported = sub {
> + my ($format, $storeid, $storecfg) = @_;
> + die "storage ID parameter must be passed to the sub" if !$storeid;
> + die "storage configuration must be passed to the sub" if !$storecfg;
> +
> + return if !$format;
> +
> + my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
> + my $supported = grep { $_ eq $format } @$valid_formats;
> +
> + die "format '$format' is not supported on storage $storeid" if !$supported;
> +};
> +
> +# storecfg ... PVE::Storage::config()
> +# vmid ... target VM ID
> +# vmconf ... target VM configuration
> +# source ... source image (volid or absolute path)
> +# target ... hash with
> +# storeid => storage ID
> +# format => disk format (optional)
> +# options => hash with device options (may or may not contain <storeid>:0)
> +# device => device where the disk is attached (for example, scsi3) (optional)
> +#
> +# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2)
> +my $import_disk_image = sub {
> + my ($storecfg, $vmid, $vmconf, $source, $target) = @_;
> + my $requested_format = $target->{format};
> + my $storeid = $target->{storeid};
> +
> + die "Source parameter is undefined!" if !defined $source;
> + $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
> +
> + eval { PVE::Storage::storage_config($storecfg, $storeid) };
> + die "Error while importing disk image $source: $@\n" if $@;
> +
> + my $src_size = PVE::Storage::file_size_info($source);
> + die "Could not get file size of $source" if !defined($src_size);
> +
> + $check_format_is_supported->($requested_format, $storeid, $storecfg);
> +
> + my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
> + $storecfg,
> + $storeid,
> + undef,
> + $requested_format,
> + );
> + my $dst_volid = PVE::Storage::vdisk_alloc(
> + $storecfg,
> + $storeid,
> + $vmid,
> + $dst_format,
> + undef,
> + $src_size / 102,
> + );
> +
> + print "Importing disk image '$source' as '$dst_volid'...\n";
> + eval {
> + local $SIG{INT} =
> + local $SIG{TERM} =
> + local $SIG{QUIT} =
> + local $SIG{HUP} =
> + local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
> +
> + my $zeroinit = PVE::Storage::volume_has_feature(
> + $storecfg,
> + 'sparseinit',
> + $dst_volid,
> + );
> + PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
> + PVE::QemuServer::qemu_img_convert(
> + $source,
> + $dst_volid,
> + $src_size,
> + undef,
> + $zeroinit,
> + );
> + PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
> +
> + };
> + if (my $err = $@) {
> + eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
> + warn "Cleanup of $dst_volid failed: $@ \n" if $@;
> +
> + die "Importing disk '$source' failed: $err\n" if $err;
> + }
> +
> + $target->{options}->{file} = $dst_volid;
> + my $options_string = PVE::QemuServer::print_drive($target->{options});
> + $target->{device} = PVE::QemuConfig->add_unused_volume($vmconf, $dst_volid)
> + if !$target->{device};
> +
> + $update_vm_api->(
> + {
> + vmid => $vmid,
> + $target->{device} => $options_string,
> + skiplock => 1,
> + digest => $vmconf->{digest},
> + },
> + 1,
> + );
> +
> + return $dst_volid;
> +};
> +
> +__PACKAGE__->register_method ({
> + name => 'importdisk',
> + path => '{vmid}/importdisk',
> + method => 'POST',
> + proxyto => 'node',
> + protected => 1,
> + description => "Import an external disk image into a VM. The image format ".
> + "has to be supported by qemu-img.",
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + vmid => get_standard_option('pve-vmid',
> + {completion => \&PVE::QemuServer::complete_vmid}),
> + source => {
> + description => "Disk image to import. Can be a volid ".
> + "(local:99/imageToImport.raw) or an absolute path on the server.",
> + type => 'string',
> + },
> + device => {
> + type => 'string',
> + description => "Bus/Device type of the new disk (e.g. 'ide0', ".
> + "'scsi2'). Will add the image as unused disk if omitted.",
> + enum => [PVE::QemuServer::Drive::valid_drive_names()],
> + optional => 1,
> + },
> + device_options => {
> + type => 'string',
> + description => "Options to set for the new disk (e.g. 'discard=on,backup=0')",
> + optional => 1,
> + requires => 'device',
> + },
> + storage => get_standard_option('pve-storage-id', {
> + description => "The storage to which the image will be imported to.",
> + completion => \&PVE::QemuServer::complete_storage,
> + }),
> + format => {
> + type => 'string',
> + description => 'Target format.',
> + enum => [ 'raw', 'qcow2', 'vmdk' ],
> + optional => 1,
> + },
> + digest => get_standard_option('pve-config-digest'),
> + },
> + },
> + returns => { type => 'string'},
> + code => sub {
> + my ($param) = @_;
> + my $vmid = extract_param($param, 'vmid');
> + my $node = extract_param($param, 'node');
> + my $source = extract_param($param, 'source');
> + my $digest = extract_param($param, 'digest');
> + my $device_options = extract_param($param, 'device_options');
> + my $device = extract_param($param, 'device');
> + my $storeid = extract_param($param, 'storage');
> +
> + my $rpcenv = PVE::RPCEnvironment::get();
> + my $authuser = $rpcenv->get_user();
> + my $storecfg = PVE::Storage::config();
> + PVE::Storage::storage_config($storecfg, $storeid);
> +
> +
> + if ($device_options) {
> + # $device_options may or may not contain <storeid>:0
> + my $parsed = PVE::QemuServer::Drive::parse_drive($device, $device_options);
> + if ($parsed) {
> + raise_param_exc({$device_options => "Invalid import syntax"})
> + if !($parsed->{file} =~ $IMPORT_DISK_RE);
> + } else {
> + my $fake = "$storeid:0,$device_options";
> + $parsed = PVE::QemuServer::Drive::parse_drive($device, $fake);
> + }
> + delete $parsed->{file};
> + delete $parsed->{interface};
> + delete $parsed->{index};
> + $device_options = $parsed;
> + }
> +
> + # Format can be set explicitly "--format vmdk"
> + # or as part of device options "--device_options discard=on,format=vmdk"
> + my $format = extract_param($param, 'format');
> + if ($device_options) {
> + raise_param_exc({format => "Format already specified in device_options!"})
> + if $format && $device_options->{format};
> + $format = $format || $device_options->{format}; # may be undefined
> + }
> + $check_format_is_supported->($format, $storeid, $storecfg);
> +
> + # quick checks before fork + lock
> + my $conf = PVE::QemuConfig->load_config($vmid);
> + PVE::QemuConfig->check_lock($conf);
> + PVE::Tools::assert_if_modified($conf->{digest}, $digest);
> + if ($device && $conf->{$device}) {
> + die "Could not import because device $device is already in ".
> + "use in VM $vmid. Choose a different device!";
> + }
> +
> + my $worker = sub {
> + PVE::QemuConfig->lock_config($vmid, sub {
> + $conf = PVE::QemuConfig->load_config($vmid);
> + PVE::QemuConfig->check_lock($conf);
> +
> + PVE::Tools::assert_if_modified($conf->{digest}, $digest);
> + PVE::QemuConfig->set_lock($vmid, 'import');
> + $conf = PVE::QemuConfig->load_config($vmid);
> + });
> +
> + my $target = {
> + node => $node,
> + storeid => $storeid,
> + };
> + $target->{format} = $format;
> + $target->{device} = $device;
> + $target->{options} = $device_options;
> + eval { $import_disk_image->($storecfg, $vmid, $conf, $source, $target) };
> + my $err = $@;
> + eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
> + warn $@ if $@;
> + die $err if $err;
> + };
> + return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker);
> + }});
> +
> +__PACKAGE__->register_method({
> + name => 'importvm',
> + path => '{vmid}/importvm',
> + method => 'POST',
> + description => "Import a VM from existing disk images.",
> + protected => 1,
> + proxyto => 'node',
> + parameters => {
> + additionalProperties => 0,
> + properties => PVE::QemuServer::json_config_properties(
> + {
> + node => get_standard_option('pve-node'),
> + vmid => get_standard_option('pve-vmid', { completion =>
> + \&PVE::Cluster::complete_next_vmid }),
> + diskimage => {
> + description => "\\0 delimited mapping of devices to disk images. For " .
> + "example, scsi0=/mnt/nfs/image1.vmdk",
> + type => 'string',
> + format => 'device-image-pair-alist',
> + },
> + start => {
> + optional => 1,
> + type => 'boolean',
> + default => 0,
> + description => "Start VM after it was imported successfully.",
> + },
> + }),
> + },
> + returns => {
> + type => 'string',
> + },
> + code => sub {
> + my ($param) = @_;
> + my $node = extract_param($param, 'node');
> + my $vmid = extract_param($param, 'vmid');
> + my $diskimages_string = extract_param($param, 'diskimage');
> + my $boot = extract_param($param, 'boot');
> + my $start = extract_param($param, 'start');
> +
> + my $rpcenv = PVE::RPCEnvironment::get();
> + my $authuser = $rpcenv->get_user();
> + my $storecfg = PVE::Storage::config();
> +
> + PVE::Cluster::check_cfs_quorum();
> +
> + my $import_param = {};
> + foreach my $opt (keys %$param) {
> + next if $opt eq 'efidisk0';
> + raise_param_exc({bootdisk => "Deprecated: Use --boot order= instead"})
> + if $opt eq 'bootdisk';
> +
> + if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
> + my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
> + if ($drive->{file} =~ $IMPORT_DISK_RE) {
> + $import_param->{$opt} = $drive;
> + delete $param->{$opt};
> + }
> + }
> + }
> +
> + my $diskimages = {};
> + foreach my $pair (PVE::Tools::split_list($diskimages_string)) {
> + my ($device, $diskimage) = split('=', $pair);
> + $diskimages->{$device} = $diskimage;
> + raise_param_exc({
> + $device => "Device '$device' not marked for import, " .
> + "but import source '$diskimage' specified",
> + }) if !defined($import_param->{$device});
> + PVE::Storage::abs_filesystem_path($storecfg, $diskimage, 1);
> + }
> +
> + foreach my $device (keys %$import_param) {
> + raise_param_exc({
> + $device => "Device '$device' marked for import, but no source given\n",
> + }) if !defined($diskimages->{$device});
> + }
> +
> + eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') };
> + die "Unable to create config for VM import: $@" if $@;
> +
> + my $worker = sub {
> + my $reload_conf = sub {
> + my ($vmid) = @_;
> + my $conf = PVE::QemuConfig->load_config($vmid);
> + return $conf if PVE::QemuConfig->has_lock($conf, 'import');
> + die "import lock in VM $vmid config file missing!";
> + };
> +
> + my $conf = $reload_conf->($vmid);
> + $update_vm_api->(
> + {
> + %$param,
> + node => $node,
> + vmid => $vmid,
> + skiplock => 1,
> + digest => $conf->{digest},
> + },
> + 1
> + );
> +
> + eval {
> + foreach my $device (keys %$import_param) {
> + $conf = $reload_conf->($vmid);
> + my $drive = $import_param->{$device};
> + my $storeid = PVE::Storage::parse_volume_id($drive->{file});
> + my $imported = $import_disk_image->(
> + $storecfg,
> + $vmid,
> + $conf,
> + $diskimages->{$device},
> + {
> + storeid => $storeid,
> + format => $drive->{format},
> + options => $drive,
> + device => $device,
> + },
> + );
> + }
> + };
> + my $err = $@;
> + if ($err) {
> + eval { PVE::QemuServer::destroy_vm($storecfg, $vmid, 1) };
> + warn "Could not destroy VM $vmid: $@" if "$@";
> +
> + die "Import failed: $err";
> + }
> +
> + $conf = $reload_conf->($vmid);
> + if (!$boot) {
> + my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf);
> + $boot = PVE::QemuServer::print_bootorder($bootdevs);
> + }
> + $update_vm_api->(
> + {
> + node => $node,
> + vmid => $vmid,
> + boot => $boot,
> + skiplock => 1,
> + digest => $conf->{digest},
> + },
> + 1,
> + );
> +
> + eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
> + warn $@ if $@;
> +
> + PVE::QemuServer::vm_start($storecfg, $vmid) if $start;
> + };
> +
> + return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker);
> + }});
> +
> +
> 1;
> diff --git a/PVE/API2/Qemu/Makefile b/PVE/API2/Qemu/Makefile
> index 5d4abda..bdd4762 100644
> --- a/PVE/API2/Qemu/Makefile
> +++ b/PVE/API2/Qemu/Makefile
> @@ -1,4 +1,4 @@
> -SOURCES=Agent.pm CPU.pm Machine.pm
> +SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm
>
> .PHONY: install
> install:
> diff --git a/PVE/API2/Qemu/OVF.pm b/PVE/API2/Qemu/OVF.pm
> new file mode 100644
> index 0000000..bd6e90b
> --- /dev/null
> +++ b/PVE/API2/Qemu/OVF.pm
> @@ -0,0 +1,68 @@
> +package PVE::API2::Qemu::OVF;
> +
> +use strict;
> +use warnings;
> +
> +use PVE::RESTHandler;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::QemuServer::OVF;
> +
> +use base qw(PVE::RESTHandler);
> +
> +__PACKAGE__->register_method ({
> + name => 'index',
> + path => '',
> + method => 'GET',
> + proxyto => 'node',
> + description => "Read an .ovf manifest.",
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + manifest => {
> + description => ".ovf manifest",
> + type => 'string',
> + },
> + },
> + },
> + returns => {
> + description => "VM config according to .ovf manifest and digest of manifest",
> + type => "object",
> + },
> + returns => {
> + type => 'object',
> + additionalProperties => 1,
> + properties => PVE::QemuServer::json_ovf_properties({
> + name => {
> + type => 'string',
> + optional => 1,
> + },
> + cores => {
> + type => 'integer',
> + optional => 1,
> + },
> + memory => {
> + type => 'integer',
> + optional => 1,
> + },
> + }),
> + },
> + code => sub {
> + my ($param) = @_;
> +
> + my $manifest = $param->{manifest};
> + die "$manifest: non-existent or non-regular file\n" if (! -f $manifest);
> +
> + my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest, 0, 1);
> + my $result;
> + $result->{cores} = $parsed->{qm}->{cores};
> + $result->{name} = $parsed->{qm}->{name};
> + $result->{memory} = $parsed->{qm}->{memory};
> + my $disks = $parsed->{disks};
> + foreach my $disk (@$disks) {
> + $result->{$disk->{disk_address}} = $disk->{backing_file};
> + }
> + return $result;
> + }});
> +
> +1;
> \ No newline at end of file
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 1c0b5c2..131c0b6 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -300,7 +300,7 @@ my $confdesc = {
> optional => 1,
> type => 'string',
> description => "Lock/unlock the VM.",
> - enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
> + enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended import)],
> },
> cpulimit => {
> optional => 1,
> @@ -985,19 +985,41 @@ PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_
> sub verify_volume_id_or_qm_path {
> my ($volid, $noerr) = @_;
>
> - if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
> - return $volid;
> - }
> + return $volid eq 'none' || $volid eq 'cdrom' ?
> + $volid :
> + verify_volume_id_or_absolute_path($volid, $noerr);
> +}
> +
> +PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
> +sub verify_volume_id_or_absolute_path {
> + my ($volid, $noerr) = @_;
> +
> + return $volid if $volid =~ m|^/|;
>
> - # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
> $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
> if ($@) {
> - return if $noerr;
> + return undef if $noerr;
> die $@;
> }
> return $volid;
> }
>
> +PVE::JSONSchema::register_format('device-image-pair', \&verify_device_image_pair);
> +sub verify_device_image_pair {
> + my ($pair, $noerr) = @_;
> +
> + my $error = sub {
> + return undef if $noerr;
> + die $@;
> + };
> +
> + my ($device, $image) = split('=', $pair);
> + $error->("Invalid device '$device'") if !PVE::QemuServer::Drive::is_valid_drivename($device);
> + $error->("Invalid image '$image'") if !verify_volume_id_or_absolute_path($image);
> +
> + return $pair;
> +}
> +
> my $usb_fmt = {
> host => {
> default_key => 1,
> @@ -2030,6 +2052,22 @@ sub json_config_properties {
> return $prop;
> }
>
> +# Properties that we can read from an OVF file
> +sub json_ovf_properties {
> + my $prop = shift;
> +
> + foreach my $device ( PVE::QemuServer::Drive::valid_drive_names()) {
> + $prop->{$device} = {
> + type => 'string',
> + format => 'pve-volume-id-or-absolute-path',
> + description => "Disk image that gets imported to $device",
> + optional => 1,
> + };
> + }
> +
> + return $prop;
> +}
> +
> # return copy of $confdesc_cloudinit to generate documentation
> sub cloudinit_config_properties {
>
> @@ -6748,7 +6786,7 @@ sub qemu_img_convert {
> $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
> $src_is_iscsi = ($src_path =~ m|^iscsi://|);
> $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
> - } elsif (-f $src_volid) {
> + } elsif (-f $src_volid || -b $src_volid) {
> $src_path = $src_volid;
> if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
> $src_format = $1;
> diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm
> index c76c199..48146e9 100644
> --- a/PVE/QemuServer/OVF.pm
> +++ b/PVE/QemuServer/OVF.pm
> @@ -87,7 +87,7 @@ sub id_to_pve {
>
> # returns two references, $qm which holds qm.conf style key/values, and \@disks
> sub parse_ovf {
> - my ($ovf, $debug) = @_;
> + my ($ovf, $debug, $manifest_only) = @_;
>
> my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1);
>
> @@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id);
> die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
> }
>
> - my $virtual_size;
> - if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
> - die "error parsing $backing_file_path, size seems to be $virtual_size\n";
> + my $virtual_size = undef;
> + if (!$manifest_only) { # Not possible if manifest is uploaded in web gui
> + if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
> + die "error parsing $backing_file_path: Could not get file size info: $@\n";
> + }
> }
>
> $pve_disk = {
> --
> 2.20.1
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel at lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
More information about the pve-devel
mailing list