[pve-devel] [pbs-devel] [PATCH proxmox-backup 2/2] file-restore: add 'tar' option to 'extract' command
Wolfgang Bumiller
w.bumiller at proxmox.com
Tue Jul 5 13:43:35 CEST 2022
needs a rebase ;-)
On Tue, May 31, 2022 at 01:17:23PM +0200, Dominik Csapak wrote:
> if the target ist stdout, we can now either have a zip (default) or a
> tar.zst (with --tar 1) by making use of the new 'format' parameter
> of the restore daemons 'extract' api
>
> Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
> ---
> proxmox-file-restore/Cargo.toml | 1 +
> proxmox-file-restore/src/block_driver.rs | 6 +--
> proxmox-file-restore/src/block_driver_qemu.rs | 4 +-
> proxmox-file-restore/src/main.rs | 51 ++++++++++++++-----
> 4 files changed, 45 insertions(+), 17 deletions(-)
>
> diff --git a/proxmox-file-restore/Cargo.toml b/proxmox-file-restore/Cargo.toml
> index 81eb7299..222a244f 100644
> --- a/proxmox-file-restore/Cargo.toml
> +++ b/proxmox-file-restore/Cargo.toml
> @@ -13,6 +13,7 @@ nix = "0.19.1"
> serde = { version = "1.0", features = ["derive"] }
> serde_json = "1.0"
> tokio = { version = "1.6", features = [ "io-std", "rt", "rt-multi-thread", "time" ] }
> +tokio-util = { version = "0.6", features = ["io"] }
>
> pxar = { version = "0.10.1", features = [ "tokio-io" ] }
>
> diff --git a/proxmox-file-restore/src/block_driver.rs b/proxmox-file-restore/src/block_driver.rs
> index 0b5face9..ed8a19d0 100644
> --- a/proxmox-file-restore/src/block_driver.rs
> +++ b/proxmox-file-restore/src/block_driver.rs
> @@ -55,7 +55,7 @@ pub trait BlockRestoreDriver {
> details: SnapRestoreDetails,
> img_file: String,
> path: Vec<u8>,
> - pxar: bool,
> + format: String,
Similarly, an enum (and possibly zstd bool) would make more sense here
IMO.
> ) -> Async<Result<Box<dyn tokio::io::AsyncRead + Unpin + Send>, Error>>;
>
> /// Return status of all running/mapped images, result value is (id, extra data), where id must
> @@ -101,10 +101,10 @@ pub async fn data_extract(
> details: SnapRestoreDetails,
> img_file: String,
> path: Vec<u8>,
> - pxar: bool,
> + format: String,
> ) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>, Error> {
> let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve();
> - driver.data_extract(details, img_file, path, pxar).await
> + driver.data_extract(details, img_file, path, format).await
> }
>
> #[api(
> diff --git a/proxmox-file-restore/src/block_driver_qemu.rs b/proxmox-file-restore/src/block_driver_qemu.rs
> index 362fff0d..dca5681e 100644
> --- a/proxmox-file-restore/src/block_driver_qemu.rs
> +++ b/proxmox-file-restore/src/block_driver_qemu.rs
> @@ -215,7 +215,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
> details: SnapRestoreDetails,
> img_file: String,
> mut path: Vec<u8>,
> - pxar: bool,
> + format: String,
> ) -> Async<Result<Box<dyn tokio::io::AsyncRead + Unpin + Send>, Error>> {
> async move {
> let client = ensure_running(&details).await?;
> @@ -228,7 +228,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
> if let Err(err) = client
> .download(
> "api2/json/extract",
> - Some(json!({ "path": path, "pxar": pxar })),
> + Some(json!({ "path": path, "format": format })),
> &mut tx,
> )
> .await
> diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
> index 3420ea8e..693da091 100644
> --- a/proxmox-file-restore/src/main.rs
> +++ b/proxmox-file-restore/src/main.rs
> @@ -4,8 +4,11 @@ use std::path::PathBuf;
> use std::sync::Arc;
>
> use anyhow::{bail, format_err, Error};
> +use futures::StreamExt;
> use serde_json::{json, Value};
> +use tokio::io::AsyncWriteExt;
>
> +use proxmox_compression::zstd::ZstdEncoder;
> use proxmox_router::cli::{
> complete_file_name, default_table_format_options, format_and_print_result_full,
> get_output_format, run_cli_command, CliCommand, CliCommandMap, CliEnvironment, ColumnConfig,
> @@ -18,7 +21,7 @@ use pxar::accessor::aio::Accessor;
> use pxar::decoder::aio::Decoder;
>
> use pbs_api_types::{BackupDir, BackupNamespace, CryptMode};
> -use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
> +use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq};
> use pbs_client::tools::{
> complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
> key_source::{
> @@ -346,9 +349,15 @@ async fn list(
> description: "Group/Snapshot path.",
> },
> "path": {
> - description: "Path to restore. Directories will be restored as .zip files if extracted to stdout.",
> + description: "Path to restore. Directories will be restored as archive files if extracted to stdout.",
> type: String,
> },
> + "tar": {
Why not stick with `format` here as well? (and the remaining hunks)
> + description: "If true, the resulting archive is a 'tar.zst' else it will be 'zip.",
> + optional: true,
> + default: false,
> + type: bool,
> + },
> "base64": {
> type: Boolean,
> description: "If set, 'path' will be interpreted as base64 encoded.",
> @@ -393,6 +402,7 @@ async fn extract(
> base64: bool,
> target: Option<String>,
> verbose: bool,
> + tar: bool,
> param: Value,
> ) -> Result<(), Error> {
> let repo = extract_repository_from_value(¶m)?;
> @@ -451,7 +461,7 @@ async fn extract(
> let archive_size = reader.archive_size();
> let reader = LocalDynamicReadAt::new(reader);
> let decoder = Accessor::new(reader, archive_size).await?;
> - extract_to_target(decoder, &path, target, verbose).await?;
> + extract_to_target(decoder, &path, target, verbose, tar).await?;
> }
> ExtractPath::VM(file, path) => {
> let details = SnapRestoreDetails {
> @@ -467,7 +477,8 @@ async fn extract(
> };
>
> if let Some(mut target) = target {
> - let reader = data_extract(driver, details, file, path.clone(), true).await?;
> + let reader =
> + data_extract(driver, details, file, path.clone(), "pxar".to_string()).await?;
> let decoder = Decoder::from_tokio(reader).await?;
> extract_sub_dir_seq(&target, decoder, verbose).await?;
>
> @@ -478,7 +489,9 @@ async fn extract(
> format_err!("unable to remove temporary .pxarexclude-cli file - {}", e)
> })?;
> } else {
> - let mut reader = data_extract(driver, details, file, path.clone(), false).await?;
> + let format = if tar { "tar.zst" } else { "zip" };
> + let mut reader =
> + data_extract(driver, details, file, path.clone(), format.to_owned()).await?;
> tokio::io::copy(&mut reader, &mut tokio::io::stdout()).await?;
> }
> }
> @@ -495,6 +508,7 @@ async fn extract_to_target<T>(
> path: &[u8],
> target: Option<PathBuf>,
> verbose: bool,
> + tar: bool,
> ) -> Result<(), Error>
> where
> T: pxar::accessor::ReadAt + Clone + Send + Sync + Unpin + 'static,
> @@ -515,13 +529,26 @@ where
> tokio::io::copy(&mut file.contents().await?, &mut tokio::io::stdout()).await?;
> }
> _ => {
> - create_zip(
> - tokio::io::stdout(),
> - decoder,
> - OsStr::from_bytes(path),
> - verbose,
> - )
> - .await?;
> + if tar {
> + let (writer, reader) = tokio::io::duplex(1024 * 1024);
> + let path = OsStr::from_bytes(path).to_owned();
> + tokio::spawn(async move { create_tar(writer, decoder, &path, verbose).await });
> + let mut zstdstream =
> + ZstdEncoder::new(tokio_util::io::ReaderStream::new(reader))?;
> + let mut stdout = tokio::io::stdout();
> + while let Some(buf) = zstdstream.next().await {
> + let buf = buf?;
> + stdout.write_all(&buf).await?;
> + }
> + } else {
> + create_zip(
> + tokio::io::stdout(),
> + decoder,
> + OsStr::from_bytes(path),
> + verbose,
> + )
> + .await?;
> + }
> }
> }
> }
> --
> 2.30.2
More information about the pve-devel
mailing list