[pve-devel] [PATCH storage] add API method to move a volume between storages
Filip Schauer
f.schauer at proxmox.com
Wed Jun 12 16:47:58 CEST 2024
I forgot to mention that this fixes #5191
On 12/06/2024 16:45, Filip Schauer wrote:
> Add the ability to move a backup, ISO, container template or snippet
> between storages of a node via an API method. Moving a VMA backup to a
> Proxmox Backup Server requires the proxmox-vma-to-pbs package to be
> installed. Currently only VMA backups can be moved to a Proxmox Backup
> Server and moving backups from a Proxmox Backup Server is not yet
> supported.
>
> The method can be called from the PVE shell with `pvesm move`:
>
> # pvesm move <source volume> <target storage>
> pvesm move local:backup/vzdump-lxc-102-2024_05_29-17_05_27.tar.zst pbs
>
> Or use curl to call the API method:
>
> curl https://$APINODE:8006/api2/json/nodes/$TARGETNODE/storage/$TARGETSTORAGE/move \
> --insecure --cookie "$(<cookie)" -H "$(<csrftoken)" -X POST \
> --data-raw "source-volume=$SOURCEVOLUME"
>
> Signed-off-by: Filip Schauer <f.schauer at proxmox.com>
> ---
> This patch depends on
> [PATCH backup-qemu/vma-to-pbs 0/2] add support for notes and logs
> https://lists.proxmox.com/pipermail/pbs-devel/2024-May/009445.html
>
> src/PVE/API2/Storage/Makefile | 2 +-
> src/PVE/API2/Storage/MoveVolume.pm | 61 +++++++++
> src/PVE/API2/Storage/Status.pm | 7 ++
> src/PVE/CLI/pvesm.pm | 33 +++++
> src/PVE/Storage.pm | 195 +++++++++++++++++++++++++----
> 5 files changed, 271 insertions(+), 27 deletions(-)
> create mode 100644 src/PVE/API2/Storage/MoveVolume.pm
>
> diff --git a/src/PVE/API2/Storage/Makefile b/src/PVE/API2/Storage/Makefile
> index 1705080..11f3c95 100644
> --- a/src/PVE/API2/Storage/Makefile
> +++ b/src/PVE/API2/Storage/Makefile
> @@ -1,5 +1,5 @@
>
> -SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm Scan.pm FileRestore.pm
> +SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm Scan.pm FileRestore.pm MoveVolume.pm
>
> .PHONY: install
> install:
> diff --git a/src/PVE/API2/Storage/MoveVolume.pm b/src/PVE/API2/Storage/MoveVolume.pm
> new file mode 100644
> index 0000000..52447a4
> --- /dev/null
> +++ b/src/PVE/API2/Storage/MoveVolume.pm
> @@ -0,0 +1,61 @@
> +package PVE::API2::Storage::MoveVolume;
> +
> +use strict;
> +use warnings;
> +
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::RESTHandler;
> +use PVE::RPCEnvironment;
> +use PVE::Storage;
> +
> +use base qw(PVE::RESTHandler);
> +
> +__PACKAGE__->register_method ({
> + name => 'move',
> + path => '',
> + method => 'POST',
> + description => "Move a volume from one storage to another",
> + permissions => {
> + description => "You need the 'Datastore.Allocate' privilege on the storages.",
> + user => 'all',
> + },
> + protected => 1,
> + proxyto => 'node',
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + 'source-volume' => {
> + description => "Source volume",
> + type => 'string',
> + },
> + 'storage' => get_standard_option('pve-storage-id', {
> + completion => \&PVE::Storage::complete_storage_enabled,
> + description => 'Target storage',
> + }),
> + },
> + },
> + returns => { type => 'string' },
> + code => sub {
> + my ($param) = @_;
> +
> + my $cfg = PVE::Storage::config();
> + my $source_volume = $param->{'source-volume'};
> + my $target_storeid = $param->{'storage'};
> +
> + my ($source_storeid, undef) = PVE::Storage::parse_volume_id($source_volume, 0);
> +
> + my $rpcenv = PVE::RPCEnvironment::get();
> + my $authuser = $rpcenv->get_user();
> +
> + $rpcenv->check($authuser, "/storage/$source_storeid", ["Datastore.Allocate"]);
> + $rpcenv->check($authuser, "/storage/$target_storeid", ["Datastore.Allocate"]);
> +
> + my $worker = sub {
> + PVE::Storage::volume_move($cfg, $source_volume, $target_storeid);
> + };
> +
> + return $rpcenv->fork_worker('move_volume', '', $authuser, $worker);
> + }});
> +
> +1;
> diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
> index dc6cc69..6c816b7 100644
> --- a/src/PVE/API2/Storage/Status.pm
> +++ b/src/PVE/API2/Storage/Status.pm
> @@ -18,6 +18,7 @@ use PVE::Tools qw(run_command);
>
> use PVE::API2::Storage::Content;
> use PVE::API2::Storage::FileRestore;
> +use PVE::API2::Storage::MoveVolume;
> use PVE::API2::Storage::PruneBackups;
> use PVE::Storage;
>
> @@ -28,6 +29,11 @@ __PACKAGE__->register_method ({
> path => '{storage}/prunebackups',
> });
>
> +__PACKAGE__->register_method ({
> + subclass => "PVE::API2::Storage::MoveVolume",
> + path => '{storage}/move',
> +});
> +
> __PACKAGE__->register_method ({
> subclass => "PVE::API2::Storage::Content",
> # set fragment delimiter (no subdirs) - we need that, because volume
> @@ -233,6 +239,7 @@ __PACKAGE__->register_method ({
> { subdir => 'rrddata' },
> { subdir => 'status' },
> { subdir => 'upload' },
> + { subdir => 'move' },
> ];
>
> return $res;
> diff --git a/src/PVE/CLI/pvesm.pm b/src/PVE/CLI/pvesm.pm
> index 9b9676b..4c042aa 100755
> --- a/src/PVE/CLI/pvesm.pm
> +++ b/src/PVE/CLI/pvesm.pm
> @@ -20,6 +20,7 @@ use PVE::Storage;
> use PVE::Tools qw(extract_param);
> use PVE::API2::Storage::Config;
> use PVE::API2::Storage::Content;
> +use PVE::API2::Storage::MoveVolume;
> use PVE::API2::Storage::PruneBackups;
> use PVE::API2::Storage::Scan;
> use PVE::API2::Storage::Status;
> @@ -480,6 +481,37 @@ __PACKAGE__->register_method ({
> }
> });
>
> +__PACKAGE__->register_method ({
> + name => 'move',
> + path => 'move',
> + method => 'POST',
> + description => "Move a volume from one storage to another",
> + protected => 1,
> + proxyto => 'node',
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + 'source-volume' => {
> + description => "Source volume",
> + type => 'string',
> + },
> + 'storage' => get_standard_option('pve-storage-id', {
> + completion => \&PVE::Storage::complete_storage_enabled,
> + description => 'Target storage',
> + }),
> + },
> + },
> + returns => { type => 'null' },
> + code => sub {
> + my ($param) = @_;
> +
> + PVE::API2::Storage::MoveVolume->move($param);
> +
> + return;
> + },
> +});
> +
> __PACKAGE__->register_method ({
> name => 'prunebackups',
> path => 'prunebackups',
> @@ -690,6 +722,7 @@ our $cmddef = {
> print "APIVER $res->{apiver}\n";
> print "APIAGE $res->{apiage}\n";
> }],
> + 'move' => [ __PACKAGE__, 'move', ['source-volume', 'storage'], { node => $nodename } ],
> 'prune-backups' => [ __PACKAGE__, 'prunebackups', ['storage'], { node => $nodename }, sub {
> my $res = shift;
>
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index f19a115..fe8a5e0 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -9,6 +9,7 @@ use IO::File;
> use IO::Socket::IP;
> use IPC::Open3;
> use File::Basename;
> +use File::Copy qw(move);
> use File::Path;
> use Cwd 'abs_path';
> use Socket;
> @@ -1620,14 +1621,20 @@ sub archive_info {
> }
>
> sub archive_remove {
> - my ($archive_path) = @_;
> + my ($archive_path, $ignore_protected) = @_;
> +
> + my $protection_path = protection_file_path($archive_path);
>
> die "cannot remove protected archive '$archive_path'\n"
> - if -e protection_file_path($archive_path);
> + if !$ignore_protected && -e $protection_path;
>
> unlink $archive_path or $! == ENOENT or die "removing archive $archive_path failed: $!\n";
>
> archive_auxiliaries_remove($archive_path);
> +
> + if (-e $protection_path) {
> + unlink $protection_path or $! == ENOENT or log_warn("Removing protection file failed: $!");
> + }
> }
>
> sub archive_auxiliaries_remove {
> @@ -1680,6 +1687,45 @@ sub extract_vzdump_config_tar {
> return wantarray ? ($raw, $file) : $raw;
> }
>
> +sub decompress_archive_into_pipe {
> + my ($archive, $cmd, $outfunc) = @_;
> +
> + my $info = archive_info($archive);
> + die "archive is not compressed\n" if !$info->{compression};
> + my $decompressor = $info->{decompressor};
> + my $full_cmd = [ [@$decompressor, $archive], $cmd ];
> +
> + # lzop/zcat exits with 1 when the pipe is closed early,
> + # detect this and ignore the exit code later
> + my $broken_pipe;
> + my $errstring;
> + my $err = sub {
> + my $output = shift;
> + if (
> + $output =~ m/lzop: Broken pipe: <stdout>/
> + || $output =~ m/gzip: stdout: Broken pipe/
> + || $output =~ m/zstd: error 70 : Write error.*Broken pipe/
> + ) {
> + $broken_pipe = 1;
> + } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
> + $errstring = "failed to decompress archive: $output\n";
> + }
> + };
> +
> + my $rc = eval { run_command($full_cmd, outfunc => $outfunc, errfunc => $err, noerr => 1) };
> + my $rerr = $@;
> +
> + $broken_pipe ||= $rc == 141; # broken pipe from cmd POV
> +
> + if (!$errstring && !$broken_pipe && $rc != 0) {
> + die "$rerr\n" if $rerr;
> + die "archive decompression failed with exit code $rc\n";
> + }
> + die "$errstring\n" if $errstring;
> +
> + return;
> +}
> +
> sub extract_vzdump_config_vma {
> my ($archive, $comp) = @_;
>
> @@ -1691,30 +1737,7 @@ sub extract_vzdump_config_vma {
> my $decompressor = $info->{decompressor};
>
> if ($comp) {
> - my $cmd = [ [@$decompressor, $archive], ["vma", "config", "-"] ];
> -
> - # lzop/zcat exits with 1 when the pipe is closed early by vma, detect this and ignore the exit code later
> - my $broken_pipe;
> - my $errstring;
> - my $err = sub {
> - my $output = shift;
> - if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error.*Broken pipe/) {
> - $broken_pipe = 1;
> - } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
> - $errstring = "Failed to extract config from VMA archive: $output\n";
> - }
> - };
> -
> - my $rc = eval { run_command($cmd, outfunc => $out, errfunc => $err, noerr => 1) };
> - my $rerr = $@;
> -
> - $broken_pipe ||= $rc == 141; # broken pipe from vma POV
> -
> - if (!$errstring && !$broken_pipe && $rc != 0) {
> - die "$rerr\n" if $rerr;
> - die "config extraction failed with exit code $rc\n";
> - }
> - die "$errstring\n" if $errstring;
> + decompress_archive_into_pipe($archive, ["vma", "config", "-"], $out);
> } else {
> run_command(["vma", "config", $archive], outfunc => $out);
> }
> @@ -1753,6 +1776,126 @@ sub extract_vzdump_config {
> }
> }
>
> +sub volume_move {
> + my ($cfg, $source_volid, $target_storeid) = @_;
> +
> + my ($source_storeid, $source_volname) = parse_volume_id($source_volid, 0);
> +
> + die "source and target storage cannot be the same\n" if ($source_storeid eq $target_storeid);
> +
> + activate_storage($cfg, $source_storeid);
> + my $source_scfg = storage_config($cfg, $source_storeid);
> + my $source_plugin = PVE::Storage::Plugin->lookup($source_scfg->{type});
> + my ($vtype) = $source_plugin->parse_volname($source_volname);
> +
> + die "source storage '$source_storeid' does not support volumes of type '$vtype'\n"
> + if !$source_scfg->{content}->{$vtype};
> +
> + activate_storage($cfg, $target_storeid);
> + my $target_scfg = storage_config($cfg, $target_storeid);
> + die "target storage '$target_storeid' does not support volumes of type '$vtype'\n"
> + if !$target_scfg->{content}->{$vtype};
> +
> + if ($vtype eq 'backup' || $vtype eq 'iso' || $vtype eq 'vztmpl' || $vtype eq 'snippets') {
> + my $target_plugin = PVE::Storage::Plugin->lookup($target_scfg->{type});
> +
> + die "moving a backup from a Proxmox Backup Server is not yet supported\n"
> + if ($vtype eq 'backup' && $source_scfg->{type} eq 'pbs');
> +
> + my $source_path = $source_plugin->filesystem_path($source_scfg, $source_volname);
> + die "$source_path does not exist" if (!-e $source_path);
> + my $source_dirname = dirname($source_path);
> +
> + return if $vtype ne 'backup';
> +
> + if ($target_scfg->{type} eq 'pbs') {
> + my $info = archive_info($source_path);
> + die "moving non-VMA backups to a Proxmox Backup Server is not yet supported\n"
> + if ($info->{format} ne 'vma');
> +
> + my $repo = PVE::PBSClient::get_repository($target_scfg);
> + my $vmid = ($source_plugin->parse_volname($source_volname))[2];
> + my $fingerprint = $target_scfg->{fingerprint};
> + my $password = PVE::Storage::PBSPlugin::pbs_password_file_name(
> + $target_scfg, $target_storeid);
> + my $namespace = $target_scfg->{namespace};
> + my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name(
> + $target_scfg, $target_storeid);
> + my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name(
> + $target_scfg, $target_storeid);
> +
> + my $comp = $info->{compression};
> + my $backup_time = $info->{ctime};
> + my $log_file_path = "$source_dirname/$info->{logfilename}";
> + my $notes_file_path = "$source_dirname/$info->{notesfilename}";
> +
> + my $vma_to_pbs_cmd = [
> + "vma-to-pbs",
> + "--repository", $repo,
> + "--vmid", $vmid,
> + "--fingerprint", $fingerprint,
> + "--password-file", $password,
> + "--backup-time", $backup_time,
> + "--compress",
> + ];
> +
> + push @$vma_to_pbs_cmd, "--ns", $namespace if $namespace;
> + push @$vma_to_pbs_cmd, "--log-file", $log_file_path if -e $log_file_path;
> + push @$vma_to_pbs_cmd, "--notes-file", $notes_file_path if -e $notes_file_path;
> + push @$vma_to_pbs_cmd, "--encrypt", "--keyfile", $keyfile if -e $keyfile;
> + push @$vma_to_pbs_cmd, "--master-keyfile", $master_keyfile if -e $master_keyfile;
> +
> + if ($comp) {
> + decompress_archive_into_pipe($source_path, $vma_to_pbs_cmd);
> + } else {
> + push @$vma_to_pbs_cmd, $source_path;
> + run_command($vma_to_pbs_cmd);
> + }
> +
> + my $protection_source_path = protection_file_path($source_path);
> +
> + if (-e $protection_source_path) {
> + my $target_volid = PVE::Storage::PBSPlugin::print_volid(
> + $target_storeid, 'vm', $vmid, $backup_time);
> + my (undef, $target_volname) = parse_volume_id($target_volid, 0);
> + $target_plugin->update_volume_attribute(
> + $target_scfg, $target_storeid, $target_volname, 'protected', 1);
> + }
> +
> + archive_remove($source_path, 1);
> + } else {
> + my $target_path = $target_plugin->filesystem_path($target_scfg, $source_volname);
> +
> + move($source_path, $target_path) or die "failed to move $vtype: $!";
> +
> + my $target_dirname = dirname($target_path);
> + my $info = archive_info($source_path);
> +
> + for my $type (qw(log notes)) {
> + my $filename = $info->{"${type}filename"} or next;
> + $source_path = "$source_dirname/$filename";
> + $target_path = "$target_dirname/$filename";
> + move($source_path, $target_path) or die "moving backup $type file failed: $!"
> + if -e $source_path;
> + }
> +
> + my $protection_source_path = protection_file_path($source_path);
> +
> + if (-e $protection_source_path) {
> + my $protection_target_path = protection_file_path($target_path);
> + move($protection_source_path, $protection_target_path)
> + or die "moving backup protection file failed: $!";
> + }
> + }
> + } elsif ($vtype eq 'images') {
> + die "use pct move-volume or qm disk move\n";
> + } elsif ($vtype eq 'rootdir') {
> + die "cannot move OpenVZ rootdir\n";
> + }
> +
> + return;
> +}
> +
> sub prune_backups {
> my ($cfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
>
More information about the pve-devel
mailing list