[pve-devel] [PATCH storage] add API method to move a volume between storages
Filip Schauer
f.schauer at proxmox.com
Wed Jun 12 16:45:49 CEST 2024
Add the ability to move a backup, ISO, container template or snippet
between storages of a node via an API method. Moving a VMA backup to a
Proxmox Backup Server requires the proxmox-vma-to-pbs package to be
installed. Currently only VMA backups can be moved to a Proxmox Backup
Server and moving backups from a Proxmox Backup Server is not yet
supported.
The method can be called from the PVE shell with `pvesm move`:
# pvesm move <source volume> <target storage>
pvesm move local:backup/vzdump-lxc-102-2024_05_29-17_05_27.tar.zst pbs
Or use curl to call the API method:
curl https://$APINODE:8006/api2/json/nodes/$TARGETNODE/storage/$TARGETSTORAGE/move \
--insecure --cookie "$(<cookie)" -H "$(<csrftoken)" -X POST \
--data-raw "source-volume=$SOURCEVOLUME"
Signed-off-by: Filip Schauer <f.schauer at proxmox.com>
---
This patch depends on
[PATCH backup-qemu/vma-to-pbs 0/2] add support for notes and logs
https://lists.proxmox.com/pipermail/pbs-devel/2024-May/009445.html
src/PVE/API2/Storage/Makefile | 2 +-
src/PVE/API2/Storage/MoveVolume.pm | 61 +++++++++
src/PVE/API2/Storage/Status.pm | 7 ++
src/PVE/CLI/pvesm.pm | 33 +++++
src/PVE/Storage.pm | 195 +++++++++++++++++++++++++----
5 files changed, 271 insertions(+), 27 deletions(-)
create mode 100644 src/PVE/API2/Storage/MoveVolume.pm
diff --git a/src/PVE/API2/Storage/Makefile b/src/PVE/API2/Storage/Makefile
index 1705080..11f3c95 100644
--- a/src/PVE/API2/Storage/Makefile
+++ b/src/PVE/API2/Storage/Makefile
@@ -1,5 +1,5 @@
-SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm Scan.pm FileRestore.pm
+SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm Scan.pm FileRestore.pm MoveVolume.pm
.PHONY: install
install:
diff --git a/src/PVE/API2/Storage/MoveVolume.pm b/src/PVE/API2/Storage/MoveVolume.pm
new file mode 100644
index 0000000..52447a4
--- /dev/null
+++ b/src/PVE/API2/Storage/MoveVolume.pm
@@ -0,0 +1,61 @@
+package PVE::API2::Storage::MoveVolume;
+
+use strict;
+use warnings;
+
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RESTHandler;
+use PVE::RPCEnvironment;
+use PVE::Storage;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+ name => 'move',
+ path => '',
+ method => 'POST',
+ description => "Move a volume from one storage to another",
+ permissions => {
+ description => "You need the 'Datastore.Allocate' privilege on the storages.",
+ user => 'all',
+ },
+ protected => 1,
+ proxyto => 'node',
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ 'source-volume' => {
+ description => "Source volume",
+ type => 'string',
+ },
+ 'storage' => get_standard_option('pve-storage-id', {
+ completion => \&PVE::Storage::complete_storage_enabled,
+ description => 'Target storage',
+ }),
+ },
+ },
+ returns => { type => 'string' },
+ code => sub {
+ my ($param) = @_;
+
+ my $cfg = PVE::Storage::config();
+ my $source_volume = $param->{'source-volume'};
+ my $target_storeid = $param->{'storage'};
+
+ my ($source_storeid, undef) = PVE::Storage::parse_volume_id($source_volume, 0);
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ $rpcenv->check($authuser, "/storage/$source_storeid", ["Datastore.Allocate"]);
+ $rpcenv->check($authuser, "/storage/$target_storeid", ["Datastore.Allocate"]);
+
+ my $worker = sub {
+ PVE::Storage::volume_move($cfg, $source_volume, $target_storeid);
+ };
+
+ return $rpcenv->fork_worker('move_volume', '', $authuser, $worker);
+ }});
+
+1;
diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
index dc6cc69..6c816b7 100644
--- a/src/PVE/API2/Storage/Status.pm
+++ b/src/PVE/API2/Storage/Status.pm
@@ -18,6 +18,7 @@ use PVE::Tools qw(run_command);
use PVE::API2::Storage::Content;
use PVE::API2::Storage::FileRestore;
+use PVE::API2::Storage::MoveVolume;
use PVE::API2::Storage::PruneBackups;
use PVE::Storage;
@@ -28,6 +29,11 @@ __PACKAGE__->register_method ({
path => '{storage}/prunebackups',
});
+__PACKAGE__->register_method ({
+ subclass => "PVE::API2::Storage::MoveVolume",
+ path => '{storage}/move',
+});
+
__PACKAGE__->register_method ({
subclass => "PVE::API2::Storage::Content",
# set fragment delimiter (no subdirs) - we need that, because volume
@@ -233,6 +239,7 @@ __PACKAGE__->register_method ({
{ subdir => 'rrddata' },
{ subdir => 'status' },
{ subdir => 'upload' },
+ { subdir => 'move' },
];
return $res;
diff --git a/src/PVE/CLI/pvesm.pm b/src/PVE/CLI/pvesm.pm
index 9b9676b..4c042aa 100755
--- a/src/PVE/CLI/pvesm.pm
+++ b/src/PVE/CLI/pvesm.pm
@@ -20,6 +20,7 @@ use PVE::Storage;
use PVE::Tools qw(extract_param);
use PVE::API2::Storage::Config;
use PVE::API2::Storage::Content;
+use PVE::API2::Storage::MoveVolume;
use PVE::API2::Storage::PruneBackups;
use PVE::API2::Storage::Scan;
use PVE::API2::Storage::Status;
@@ -480,6 +481,37 @@ __PACKAGE__->register_method ({
}
});
+__PACKAGE__->register_method ({
+ name => 'move',
+ path => 'move',
+ method => 'POST',
+ description => "Move a volume from one storage to another",
+ protected => 1,
+ proxyto => 'node',
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ 'source-volume' => {
+ description => "Source volume",
+ type => 'string',
+ },
+ 'storage' => get_standard_option('pve-storage-id', {
+ completion => \&PVE::Storage::complete_storage_enabled,
+ description => 'Target storage',
+ }),
+ },
+ },
+ returns => { type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ PVE::API2::Storage::MoveVolume->move($param);
+
+ return;
+ },
+});
+
__PACKAGE__->register_method ({
name => 'prunebackups',
path => 'prunebackups',
@@ -690,6 +722,7 @@ our $cmddef = {
print "APIVER $res->{apiver}\n";
print "APIAGE $res->{apiage}\n";
}],
+ 'move' => [ __PACKAGE__, 'move', ['source-volume', 'storage'], { node => $nodename } ],
'prune-backups' => [ __PACKAGE__, 'prunebackups', ['storage'], { node => $nodename }, sub {
my $res = shift;
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index f19a115..fe8a5e0 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -9,6 +9,7 @@ use IO::File;
use IO::Socket::IP;
use IPC::Open3;
use File::Basename;
+use File::Copy qw(move);
use File::Path;
use Cwd 'abs_path';
use Socket;
@@ -1620,14 +1621,20 @@ sub archive_info {
}
sub archive_remove {
- my ($archive_path) = @_;
+ my ($archive_path, $ignore_protected) = @_;
+
+ my $protection_path = protection_file_path($archive_path);
die "cannot remove protected archive '$archive_path'\n"
- if -e protection_file_path($archive_path);
+ if !$ignore_protected && -e $protection_path;
unlink $archive_path or $! == ENOENT or die "removing archive $archive_path failed: $!\n";
archive_auxiliaries_remove($archive_path);
+
+ if (-e $protection_path) {
+ unlink $protection_path or $! == ENOENT or log_warn("Removing protection file failed: $!");
+ }
}
sub archive_auxiliaries_remove {
@@ -1680,6 +1687,45 @@ sub extract_vzdump_config_tar {
return wantarray ? ($raw, $file) : $raw;
}
+sub decompress_archive_into_pipe {
+ my ($archive, $cmd, $outfunc) = @_;
+
+ my $info = archive_info($archive);
+ die "archive is not compressed\n" if !$info->{compression};
+ my $decompressor = $info->{decompressor};
+ my $full_cmd = [ [@$decompressor, $archive], $cmd ];
+
+ # lzop/zcat exits with 1 when the pipe is closed early,
+ # detect this and ignore the exit code later
+ my $broken_pipe;
+ my $errstring;
+ my $err = sub {
+ my $output = shift;
+ if (
+ $output =~ m/lzop: Broken pipe: <stdout>/
+ || $output =~ m/gzip: stdout: Broken pipe/
+ || $output =~ m/zstd: error 70 : Write error.*Broken pipe/
+ ) {
+ $broken_pipe = 1;
+ } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
+ $errstring = "failed to decompress archive: $output\n";
+ }
+ };
+
+ my $rc = eval { run_command($full_cmd, outfunc => $outfunc, errfunc => $err, noerr => 1) };
+ my $rerr = $@;
+
+ $broken_pipe ||= $rc == 141; # broken pipe from cmd POV
+
+ if (!$errstring && !$broken_pipe && $rc != 0) {
+ die "$rerr\n" if $rerr;
+ die "archive decompression failed with exit code $rc\n";
+ }
+ die "$errstring\n" if $errstring;
+
+ return;
+}
+
sub extract_vzdump_config_vma {
my ($archive, $comp) = @_;
@@ -1691,30 +1737,7 @@ sub extract_vzdump_config_vma {
my $decompressor = $info->{decompressor};
if ($comp) {
- my $cmd = [ [@$decompressor, $archive], ["vma", "config", "-"] ];
-
- # lzop/zcat exits with 1 when the pipe is closed early by vma, detect this and ignore the exit code later
- my $broken_pipe;
- my $errstring;
- my $err = sub {
- my $output = shift;
- if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error.*Broken pipe/) {
- $broken_pipe = 1;
- } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
- $errstring = "Failed to extract config from VMA archive: $output\n";
- }
- };
-
- my $rc = eval { run_command($cmd, outfunc => $out, errfunc => $err, noerr => 1) };
- my $rerr = $@;
-
- $broken_pipe ||= $rc == 141; # broken pipe from vma POV
-
- if (!$errstring && !$broken_pipe && $rc != 0) {
- die "$rerr\n" if $rerr;
- die "config extraction failed with exit code $rc\n";
- }
- die "$errstring\n" if $errstring;
+ decompress_archive_into_pipe($archive, ["vma", "config", "-"], $out);
} else {
run_command(["vma", "config", $archive], outfunc => $out);
}
@@ -1753,6 +1776,126 @@ sub extract_vzdump_config {
}
}
+sub volume_move {
+ my ($cfg, $source_volid, $target_storeid) = @_;
+
+ my ($source_storeid, $source_volname) = parse_volume_id($source_volid, 0);
+
+ die "source and target storage cannot be the same\n" if ($source_storeid eq $target_storeid);
+
+ activate_storage($cfg, $source_storeid);
+ my $source_scfg = storage_config($cfg, $source_storeid);
+ my $source_plugin = PVE::Storage::Plugin->lookup($source_scfg->{type});
+ my ($vtype) = $source_plugin->parse_volname($source_volname);
+
+ die "source storage '$source_storeid' does not support volumes of type '$vtype'\n"
+ if !$source_scfg->{content}->{$vtype};
+
+ activate_storage($cfg, $target_storeid);
+ my $target_scfg = storage_config($cfg, $target_storeid);
+ die "target storage '$target_storeid' does not support volumes of type '$vtype'\n"
+ if !$target_scfg->{content}->{$vtype};
+
+ if ($vtype eq 'backup' || $vtype eq 'iso' || $vtype eq 'vztmpl' || $vtype eq 'snippets') {
+ my $target_plugin = PVE::Storage::Plugin->lookup($target_scfg->{type});
+
+ die "moving a backup from a Proxmox Backup Server is not yet supported\n"
+ if ($vtype eq 'backup' && $source_scfg->{type} eq 'pbs');
+
+ my $source_path = $source_plugin->filesystem_path($source_scfg, $source_volname);
+ die "$source_path does not exist" if (!-e $source_path);
+ my $source_dirname = dirname($source_path);
+
+ return if $vtype ne 'backup';
+
+ if ($target_scfg->{type} eq 'pbs') {
+ my $info = archive_info($source_path);
+ die "moving non-VMA backups to a Proxmox Backup Server is not yet supported\n"
+ if ($info->{format} ne 'vma');
+
+ my $repo = PVE::PBSClient::get_repository($target_scfg);
+ my $vmid = ($source_plugin->parse_volname($source_volname))[2];
+ my $fingerprint = $target_scfg->{fingerprint};
+ my $password = PVE::Storage::PBSPlugin::pbs_password_file_name(
+ $target_scfg, $target_storeid);
+ my $namespace = $target_scfg->{namespace};
+ my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name(
+ $target_scfg, $target_storeid);
+ my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name(
+ $target_scfg, $target_storeid);
+
+ my $comp = $info->{compression};
+ my $backup_time = $info->{ctime};
+ my $log_file_path = "$source_dirname/$info->{logfilename}";
+ my $notes_file_path = "$source_dirname/$info->{notesfilename}";
+
+ my $vma_to_pbs_cmd = [
+ "vma-to-pbs",
+ "--repository", $repo,
+ "--vmid", $vmid,
+ "--fingerprint", $fingerprint,
+ "--password-file", $password,
+ "--backup-time", $backup_time,
+ "--compress",
+ ];
+
+ push @$vma_to_pbs_cmd, "--ns", $namespace if $namespace;
+ push @$vma_to_pbs_cmd, "--log-file", $log_file_path if -e $log_file_path;
+ push @$vma_to_pbs_cmd, "--notes-file", $notes_file_path if -e $notes_file_path;
+ push @$vma_to_pbs_cmd, "--encrypt", "--keyfile", $keyfile if -e $keyfile;
+ push @$vma_to_pbs_cmd, "--master-keyfile", $master_keyfile if -e $master_keyfile;
+
+ if ($comp) {
+ decompress_archive_into_pipe($source_path, $vma_to_pbs_cmd);
+ } else {
+ push @$vma_to_pbs_cmd, $source_path;
+ run_command($vma_to_pbs_cmd);
+ }
+
+ my $protection_source_path = protection_file_path($source_path);
+
+ if (-e $protection_source_path) {
+ my $target_volid = PVE::Storage::PBSPlugin::print_volid(
+ $target_storeid, 'vm', $vmid, $backup_time);
+ my (undef, $target_volname) = parse_volume_id($target_volid, 0);
+ $target_plugin->update_volume_attribute(
+ $target_scfg, $target_storeid, $target_volname, 'protected', 1);
+ }
+
+ archive_remove($source_path, 1);
+ } else {
+ my $target_path = $target_plugin->filesystem_path($target_scfg, $source_volname);
+
+ move($source_path, $target_path) or die "failed to move $vtype: $!";
+
+ my $target_dirname = dirname($target_path);
+ my $info = archive_info($source_path);
+
+ for my $type (qw(log notes)) {
+ my $filename = $info->{"${type}filename"} or next;
+ $source_path = "$source_dirname/$filename";
+ $target_path = "$target_dirname/$filename";
+ move($source_path, $target_path) or die "moving backup $type file failed: $!"
+ if -e $source_path;
+ }
+
+ my $protection_source_path = protection_file_path($source_path);
+
+ if (-e $protection_source_path) {
+ my $protection_target_path = protection_file_path($target_path);
+ move($protection_source_path, $protection_target_path)
+ or die "moving backup protection file failed: $!";
+ }
+ }
+ } elsif ($vtype eq 'images') {
+ die "use pct move-volume or qm disk move\n";
+ } elsif ($vtype eq 'rootdir') {
+ die "cannot move OpenVZ rootdir\n";
+ }
+
+ return;
+}
+
sub prune_backups {
my ($cfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_;
--
2.39.2
More information about the pve-devel
mailing list