[pve-devel] [pbs-devel] [PATCH proxmox-backup 2/2] file-restore: add 'tar' option to 'extract' command

Wolfgang Bumiller w.bumiller at proxmox.com
Tue Jul 5 13:43:35 CEST 2022


needs a rebase ;-)

On Tue, May 31, 2022 at 01:17:23PM +0200, Dominik Csapak wrote:
> if the target ist stdout, we can now either have a zip (default) or a
> tar.zst (with --tar 1) by making use of the new 'format' parameter
> of the restore daemons 'extract' api
> 
> Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
> ---
>  proxmox-file-restore/Cargo.toml               |  1 +
>  proxmox-file-restore/src/block_driver.rs      |  6 +--
>  proxmox-file-restore/src/block_driver_qemu.rs |  4 +-
>  proxmox-file-restore/src/main.rs              | 51 ++++++++++++++-----
>  4 files changed, 45 insertions(+), 17 deletions(-)
> 
> diff --git a/proxmox-file-restore/Cargo.toml b/proxmox-file-restore/Cargo.toml
> index 81eb7299..222a244f 100644
> --- a/proxmox-file-restore/Cargo.toml
> +++ b/proxmox-file-restore/Cargo.toml
> @@ -13,6 +13,7 @@ nix = "0.19.1"
>  serde = { version = "1.0", features = ["derive"] }
>  serde_json = "1.0"
>  tokio = { version = "1.6", features = [ "io-std", "rt", "rt-multi-thread", "time" ] }
> +tokio-util = { version = "0.6", features = ["io"] }
>  
>  pxar = { version = "0.10.1", features = [ "tokio-io" ] }
>  
> diff --git a/proxmox-file-restore/src/block_driver.rs b/proxmox-file-restore/src/block_driver.rs
> index 0b5face9..ed8a19d0 100644
> --- a/proxmox-file-restore/src/block_driver.rs
> +++ b/proxmox-file-restore/src/block_driver.rs
> @@ -55,7 +55,7 @@ pub trait BlockRestoreDriver {
>          details: SnapRestoreDetails,
>          img_file: String,
>          path: Vec<u8>,
> -        pxar: bool,
> +        format: String,

Similarly, an enum (and possibly zstd bool) would make more sense here
IMO.

>      ) -> Async<Result<Box<dyn tokio::io::AsyncRead + Unpin + Send>, Error>>;
>  
>      /// Return status of all running/mapped images, result value is (id, extra data), where id must
> @@ -101,10 +101,10 @@ pub async fn data_extract(
>      details: SnapRestoreDetails,
>      img_file: String,
>      path: Vec<u8>,
> -    pxar: bool,
> +    format: String,
>  ) -> Result<Box<dyn tokio::io::AsyncRead + Send + Unpin>, Error> {
>      let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve();
> -    driver.data_extract(details, img_file, path, pxar).await
> +    driver.data_extract(details, img_file, path, format).await
>  }
>  
>  #[api(
> diff --git a/proxmox-file-restore/src/block_driver_qemu.rs b/proxmox-file-restore/src/block_driver_qemu.rs
> index 362fff0d..dca5681e 100644
> --- a/proxmox-file-restore/src/block_driver_qemu.rs
> +++ b/proxmox-file-restore/src/block_driver_qemu.rs
> @@ -215,7 +215,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
>          details: SnapRestoreDetails,
>          img_file: String,
>          mut path: Vec<u8>,
> -        pxar: bool,
> +        format: String,
>      ) -> Async<Result<Box<dyn tokio::io::AsyncRead + Unpin + Send>, Error>> {
>          async move {
>              let client = ensure_running(&details).await?;
> @@ -228,7 +228,7 @@ impl BlockRestoreDriver for QemuBlockDriver {
>                  if let Err(err) = client
>                      .download(
>                          "api2/json/extract",
> -                        Some(json!({ "path": path, "pxar": pxar })),
> +                        Some(json!({ "path": path, "format": format })),
>                          &mut tx,
>                      )
>                      .await
> diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
> index 3420ea8e..693da091 100644
> --- a/proxmox-file-restore/src/main.rs
> +++ b/proxmox-file-restore/src/main.rs
> @@ -4,8 +4,11 @@ use std::path::PathBuf;
>  use std::sync::Arc;
>  
>  use anyhow::{bail, format_err, Error};
> +use futures::StreamExt;
>  use serde_json::{json, Value};
> +use tokio::io::AsyncWriteExt;
>  
> +use proxmox_compression::zstd::ZstdEncoder;
>  use proxmox_router::cli::{
>      complete_file_name, default_table_format_options, format_and_print_result_full,
>      get_output_format, run_cli_command, CliCommand, CliCommandMap, CliEnvironment, ColumnConfig,
> @@ -18,7 +21,7 @@ use pxar::accessor::aio::Accessor;
>  use pxar::decoder::aio::Decoder;
>  
>  use pbs_api_types::{BackupDir, BackupNamespace, CryptMode};
> -use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq};
> +use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq};
>  use pbs_client::tools::{
>      complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
>      key_source::{
> @@ -346,9 +349,15 @@ async fn list(
>                  description: "Group/Snapshot path.",
>              },
>              "path": {
> -                description: "Path to restore. Directories will be restored as .zip files if extracted to stdout.",
> +                description: "Path to restore. Directories will be restored as archive files if extracted to stdout.",
>                  type: String,
>              },
> +            "tar": {

Why not stick with `format` here as well? (and the remaining hunks)

> +                description: "If true, the resulting archive is a 'tar.zst' else it will be 'zip.",
> +                optional: true,
> +                default: false,
> +                type: bool,
> +            },
>              "base64": {
>                  type: Boolean,
>                  description: "If set, 'path' will be interpreted as base64 encoded.",
> @@ -393,6 +402,7 @@ async fn extract(
>      base64: bool,
>      target: Option<String>,
>      verbose: bool,
> +    tar: bool,
>      param: Value,
>  ) -> Result<(), Error> {
>      let repo = extract_repository_from_value(&param)?;
> @@ -451,7 +461,7 @@ async fn extract(
>              let archive_size = reader.archive_size();
>              let reader = LocalDynamicReadAt::new(reader);
>              let decoder = Accessor::new(reader, archive_size).await?;
> -            extract_to_target(decoder, &path, target, verbose).await?;
> +            extract_to_target(decoder, &path, target, verbose, tar).await?;
>          }
>          ExtractPath::VM(file, path) => {
>              let details = SnapRestoreDetails {
> @@ -467,7 +477,8 @@ async fn extract(
>              };
>  
>              if let Some(mut target) = target {
> -                let reader = data_extract(driver, details, file, path.clone(), true).await?;
> +                let reader =
> +                    data_extract(driver, details, file, path.clone(), "pxar".to_string()).await?;
>                  let decoder = Decoder::from_tokio(reader).await?;
>                  extract_sub_dir_seq(&target, decoder, verbose).await?;
>  
> @@ -478,7 +489,9 @@ async fn extract(
>                      format_err!("unable to remove temporary .pxarexclude-cli file - {}", e)
>                  })?;
>              } else {
> -                let mut reader = data_extract(driver, details, file, path.clone(), false).await?;
> +                let format = if tar { "tar.zst" } else { "zip" };
> +                let mut reader =
> +                    data_extract(driver, details, file, path.clone(), format.to_owned()).await?;
>                  tokio::io::copy(&mut reader, &mut tokio::io::stdout()).await?;
>              }
>          }
> @@ -495,6 +508,7 @@ async fn extract_to_target<T>(
>      path: &[u8],
>      target: Option<PathBuf>,
>      verbose: bool,
> +    tar: bool,
>  ) -> Result<(), Error>
>  where
>      T: pxar::accessor::ReadAt + Clone + Send + Sync + Unpin + 'static,
> @@ -515,13 +529,26 @@ where
>                  tokio::io::copy(&mut file.contents().await?, &mut tokio::io::stdout()).await?;
>              }
>              _ => {
> -                create_zip(
> -                    tokio::io::stdout(),
> -                    decoder,
> -                    OsStr::from_bytes(path),
> -                    verbose,
> -                )
> -                .await?;
> +                if tar {
> +                    let (writer, reader) = tokio::io::duplex(1024 * 1024);
> +                    let path = OsStr::from_bytes(path).to_owned();
> +                    tokio::spawn(async move { create_tar(writer, decoder, &path, verbose).await });
> +                    let mut zstdstream =
> +                        ZstdEncoder::new(tokio_util::io::ReaderStream::new(reader))?;
> +                    let mut stdout = tokio::io::stdout();
> +                    while let Some(buf) = zstdstream.next().await {
> +                        let buf = buf?;
> +                        stdout.write_all(&buf).await?;
> +                    }
> +                } else {
> +                    create_zip(
> +                        tokio::io::stdout(),
> +                        decoder,
> +                        OsStr::from_bytes(path),
> +                        verbose,
> +                    )
> +                    .await?;
> +                }
>              }
>          }
>      }
> -- 
> 2.30.2





More information about the pve-devel mailing list