[pbs-devel] [PATCH proxmox-backup 09/15] pxar: factor out PxarCreateOptions
Fabian Grünbichler
f.gruenbichler at proxmox.com
Mon Jan 25 14:42:54 CET 2021
containing the CLI parameters that are mostly passed-through from the
client to our pxar archive creation wrapper in pxar::create
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
src/bin/proxmox-backup-client.rs | 27 ++++++++++++---------------
src/bin/pxar.rs | 22 ++++++++++++++--------
src/client/pxar_backup_stream.rs | 28 ++++++----------------------
src/pxar/create.rs | 28 ++++++++++++++++++++++------
src/pxar/mod.rs | 2 +-
tests/catar.rs | 10 ++++++----
6 files changed, 61 insertions(+), 56 deletions(-)
diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs
index af3b8464..c3d6a0ed 100644
--- a/src/bin/proxmox-backup-client.rs
+++ b/src/bin/proxmox-backup-client.rs
@@ -283,23 +283,15 @@ async fn backup_directory<P: AsRef<Path>>(
dir_path: P,
archive_name: &str,
chunk_size: Option<usize>,
- device_set: Option<HashSet<u64>>,
- verbose: bool,
- skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
- exclude_pattern: Vec<MatchEntry>,
- entries_max: usize,
+ pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
upload_options: UploadOptions,
) -> Result<BackupStats, Error> {
let pxar_stream = PxarBackupStream::open(
dir_path.as_ref(),
- device_set,
- verbose,
- skip_lost_and_found,
catalog,
- exclude_pattern,
- entries_max,
+ pxar_create_options,
)?;
let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
@@ -1039,6 +1031,15 @@ async fn create_backup(
println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
+
+ let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
+ device_set: devices.clone(),
+ patterns: pattern_list.clone(),
+ entries_max: entries_max as usize,
+ skip_lost_and_found,
+ verbose,
+ };
+
let upload_options = UploadOptions {
previous_manifest: previous_manifest.clone(),
compress: true,
@@ -1051,12 +1052,8 @@ async fn create_backup(
&filename,
&target,
chunk_size_opt,
- devices.clone(),
- verbose,
- skip_lost_and_found,
catalog.clone(),
- pattern_list.clone(),
- entries_max as usize,
+ pxar_options,
upload_options,
).await?;
manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;
diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs
index 3d92dbe0..85606c1d 100644
--- a/src/bin/pxar.rs
+++ b/src/bin/pxar.rs
@@ -311,16 +311,16 @@ fn create_archive(
exclude: Option<Vec<String>>,
entries_max: isize,
) -> Result<(), Error> {
- let pattern_list = {
+ let patterns = {
let input = exclude.unwrap_or_else(Vec::new);
- let mut pattern_list = Vec::with_capacity(input.len());
+ let mut patterns = Vec::with_capacity(input.len());
for entry in input {
- pattern_list.push(
+ patterns.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
.map_err(|err| format_err!("error in exclude pattern: {}", err))?,
);
}
- pattern_list
+ patterns
};
let device_set = if all_file_systems {
@@ -329,6 +329,15 @@ fn create_archive(
Some(HashSet::new())
};
+ let options = proxmox_backup::pxar::PxarCreateOptions {
+ entries_max: entries_max as usize,
+ device_set,
+ patterns,
+ verbose,
+ skip_lost_and_found: false,
+ };
+
+
let source = PathBuf::from(source);
let dir = nix::dir::Dir::open(
@@ -368,18 +377,15 @@ fn create_archive(
proxmox_backup::pxar::create_archive(
dir,
writer,
- pattern_list,
feature_flags,
- device_set,
- false,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
- entries_max as usize,
None,
+ options,
)?;
Ok(())
diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs
index aa3355fe..5fb28fd5 100644
--- a/src/client/pxar_backup_stream.rs
+++ b/src/client/pxar_backup_stream.rs
@@ -1,4 +1,3 @@
-use std::collections::HashSet;
use std::io::Write;
//use std::os::unix::io::FromRawFd;
use std::path::Path;
@@ -13,8 +12,6 @@ use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
-use pathpatterns::MatchEntry;
-
use crate::backup::CatalogWriter;
/// Stream implementation to encode and upload .pxar archives.
@@ -38,12 +35,8 @@ impl Drop for PxarBackupStream {
impl PxarBackupStream {
pub fn new<W: Write + Send + 'static>(
dir: Dir,
- device_set: Option<HashSet<u64>>,
- verbose: bool,
- skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
- patterns: Vec<MatchEntry>,
- entries_max: usize,
+ options: crate::pxar::PxarCreateOptions,
) -> Result<Self, Error> {
let (tx, rx) = std::sync::mpsc::sync_channel(10);
@@ -61,22 +54,21 @@ impl PxarBackupStream {
crate::tools::StdChannelWriter::new(tx),
);
+ let verbose = options.verbose;
+
let writer = pxar::encoder::sync::StandardWriter::new(writer);
if let Err(err) = crate::pxar::create_archive(
dir,
writer,
- patterns,
crate::pxar::Flags::DEFAULT,
- device_set,
- skip_lost_and_found,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
- entries_max,
Some(&mut *catalog_guard),
+ options,
) {
let mut error = error.lock().unwrap();
*error = Some(err.to_string());
@@ -93,23 +85,15 @@ impl PxarBackupStream {
pub fn open<W: Write + Send + 'static>(
dirname: &Path,
- device_set: Option<HashSet<u64>>,
- verbose: bool,
- skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
- patterns: Vec<MatchEntry>,
- entries_max: usize,
+ options: crate::pxar::PxarCreateOptions,
) -> Result<Self, Error> {
let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
Self::new(
dir,
- device_set,
- verbose,
- skip_lost_and_found,
catalog,
- patterns,
- entries_max,
+ options,
)
}
}
diff --git a/src/pxar/create.rs b/src/pxar/create.rs
index a21511c3..229e3f59 100644
--- a/src/pxar/create.rs
+++ b/src/pxar/create.rs
@@ -27,6 +27,22 @@ use crate::pxar::Flags;
use crate::pxar::tools::assert_single_path_component;
use crate::tools::{acl, fs, xattr, Fd};
+/// Pxar options for creating a pxar archive/stream
+#[derive(Default, Clone)]
+pub struct PxarCreateOptions {
+ /// Device/mountpoint st_dev numbers that should be included. None for no limitation.
+ pub device_set: Option<HashSet<u64>>,
+ /// Exclusion patterns
+ pub patterns: Vec<MatchEntry>,
+ /// Maximum number of entries to hold in memory
+ pub entries_max: usize,
+ /// Skip lost+found directory
+ pub skip_lost_and_found: bool,
+ /// Verbose output
+ pub verbose: bool,
+}
+
+
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
let mut fs_stat = std::mem::MaybeUninit::uninit();
let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
@@ -136,13 +152,10 @@ type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::Seq
pub fn create_archive<T, F>(
source_dir: Dir,
mut writer: T,
- mut patterns: Vec<MatchEntry>,
feature_flags: Flags,
- mut device_set: Option<HashSet<u64>>,
- skip_lost_and_found: bool,
mut callback: F,
- entry_limit: usize,
catalog: Option<&mut dyn BackupCatalogWriter>,
+ options: PxarCreateOptions,
) -> Result<(), Error>
where
T: pxar::encoder::SeqWrite,
@@ -164,6 +177,7 @@ where
)
.map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
+ let mut device_set = options.device_set.clone();
if let Some(ref mut set) = device_set {
set.insert(stat.st_dev);
}
@@ -171,7 +185,9 @@ where
let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
let mut encoder = Encoder::new(writer, &metadata)?;
- if skip_lost_and_found {
+ let mut patterns = options.patterns.clone();
+
+ if options.skip_lost_and_found {
patterns.push(MatchEntry::parse_pattern(
"lost+found",
PatternFlag::PATH_NAME,
@@ -188,7 +204,7 @@ where
catalog,
path: PathBuf::new(),
entry_counter: 0,
- entry_limit,
+ entry_limit: options.entries_max,
current_st_dev: stat.st_dev,
device_set,
hardlinks: HashMap::new(),
diff --git a/src/pxar/mod.rs b/src/pxar/mod.rs
index 82998cf8..c10bb6fb 100644
--- a/src/pxar/mod.rs
+++ b/src/pxar/mod.rs
@@ -58,7 +58,7 @@ pub(crate) mod tools;
mod flags;
pub use flags::Flags;
-pub use create::create_archive;
+pub use create::{create_archive, PxarCreateOptions};
pub use extract::{extract_archive, ErrorHandler};
/// The format requires to build sorted directory lookup tables in
diff --git a/tests/catar.rs b/tests/catar.rs
index a6acf9af..2d9dea71 100644
--- a/tests/catar.rs
+++ b/tests/catar.rs
@@ -25,16 +25,18 @@ fn run_test(dir_name: &str) -> Result<(), Error> {
dir_name, nix::fcntl::OFlag::O_NOFOLLOW,
nix::sys::stat::Mode::empty())?;
+ let options = PxarCreateOptions {
+ entries_max: ENCODER_MAX_ENTRIES,
+ ..PxarCreateOptions::default()
+ };
+
create_archive(
dir,
writer,
- Vec::new(),
Flags::DEFAULT,
- None,
- false,
|_| Ok(()),
- ENCODER_MAX_ENTRIES,
None,
+ options,
)?;
Command::new("cmp")
--
2.20.1
More information about the pbs-devel
mailing list