[pbs-devel] [PATCH v5 proxmox-backup 15/62] client: pxar: switch to stack based encoder state
Christian Ebner
c.ebner at proxmox.com
Tue May 7 17:51:57 CEST 2024
... and add the new optional encoder/decoder/accessor parameter to
attach a dedicated payload input/output.
In preparation for look-ahead caching, where a passing around of
per-directory level encoder instances with internal references is
not feasible.
Previously, for each directory level a new encoder instance has been
generated, restricting possible implementation errors. These encoder
instances have been internally linked by references to keep track of
the state changes in a parent child relationship.
This is however not feasible when the encoder has to be passed by
mutable reference, as required by the look-ahead cache
implementation. The encoder has therefore been adapted to use a
single instance implementation with an internal stack keeping track
of the state.
Depends on the bumped pxar library version, including the patches to
optionally attach the dedicated payload input/output.
Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
changes since version 4:
- no changes
pbs-client/src/pxar/create.rs | 8 +++++---
pbs-pxar-fuse/src/lib.rs | 2 +-
proxmox-backup-client/src/catalog.rs | 2 +-
proxmox-backup-client/src/main.rs | 2 +-
proxmox-backup-client/src/mount.rs | 2 +-
proxmox-file-restore/src/main.rs | 4 ++--
pxar-bin/src/main.rs | 2 +-
src/api2/admin/datastore.rs | 2 +-
src/api2/tape/restore.rs | 4 ++--
src/bin/proxmox_backup_debug/diff.rs | 2 +-
src/tape/file_formats/snapshot_archive.rs | 3 ++-
11 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index 60efb0ce5..c9bf6df85 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -170,7 +170,7 @@ where
set.insert(stat.st_dev);
}
- let mut encoder = Encoder::new(&mut writer, &metadata).await?;
+ let mut encoder = Encoder::new(&mut writer, &metadata, None).await?;
let mut patterns = options.patterns;
@@ -203,6 +203,8 @@ where
.archive_dir_contents(&mut encoder, source_dir, true)
.await?;
encoder.finish().await?;
+ encoder.close().await?;
+
Ok(())
}
@@ -663,7 +665,7 @@ impl Archiver {
) -> Result<(), Error> {
let dir_name = OsStr::from_bytes(dir_name.to_bytes());
- let mut encoder = encoder.create_directory(dir_name, metadata).await?;
+ encoder.create_directory(dir_name, metadata).await?;
let old_fs_magic = self.fs_magic;
let old_fs_feature_flags = self.fs_feature_flags;
@@ -686,7 +688,7 @@ impl Archiver {
log::info!("skipping mount point: {:?}", self.path);
Ok(())
} else {
- self.archive_dir_contents(&mut encoder, dir, false).await
+ self.archive_dir_contents(encoder, dir, false).await
};
self.fs_magic = old_fs_magic;
diff --git a/pbs-pxar-fuse/src/lib.rs b/pbs-pxar-fuse/src/lib.rs
index bf196b6c4..dff7aac31 100644
--- a/pbs-pxar-fuse/src/lib.rs
+++ b/pbs-pxar-fuse/src/lib.rs
@@ -66,7 +66,7 @@ impl Session {
let file = std::fs::File::open(archive_path)?;
let file_size = file.metadata()?.len();
let reader: Reader = Arc::new(accessor::sync::FileReader::new(file));
- let accessor = Accessor::new(reader, file_size).await?;
+ let accessor = Accessor::new(reader, file_size, None).await?;
Self::mount(accessor, options, verbose, mountpoint)
}
diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs
index 72b22e67f..db919477f 100644
--- a/proxmox-backup-client/src/catalog.rs
+++ b/proxmox-backup-client/src/catalog.rs
@@ -220,7 +220,7 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader: pbs_pxar_fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader));
- let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size).await?;
+ let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size, None).await?;
client.download(CATALOG_NAME, &mut tmpfile).await?;
let index = DynamicIndexReader::new(tmpfile)
diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index 32fe914c4..287005024 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -1453,7 +1453,7 @@ async fn restore(
if let Some(target) = target {
pbs_client::pxar::extract_archive(
- pxar::decoder::Decoder::from_std(reader)?,
+ pxar::decoder::Decoder::from_std(reader, None)?,
Path::new(target),
feature_flags,
|path| {
diff --git a/proxmox-backup-client/src/mount.rs b/proxmox-backup-client/src/mount.rs
index 4a2f83357..67fd23468 100644
--- a/proxmox-backup-client/src/mount.rs
+++ b/proxmox-backup-client/src/mount.rs
@@ -296,7 +296,7 @@ async fn mount_do(param: Value, pipe: Option<OwnedFd>) -> Result<Value, Error> {
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader: pbs_pxar_fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader));
- let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size).await?;
+ let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size, None).await?;
let session =
pbs_pxar_fuse::Session::mount(decoder, options, false, Path::new(target.unwrap()))
diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
index 50875a636..dbab69942 100644
--- a/proxmox-file-restore/src/main.rs
+++ b/proxmox-file-restore/src/main.rs
@@ -457,7 +457,7 @@ async fn extract(
let archive_size = reader.archive_size();
let reader = LocalDynamicReadAt::new(reader);
- let decoder = Accessor::new(reader, archive_size).await?;
+ let decoder = Accessor::new(reader, archive_size, None).await?;
extract_to_target(decoder, &path, target, format, zstd).await?;
}
ExtractPath::VM(file, path) => {
@@ -483,7 +483,7 @@ async fn extract(
false,
)
.await?;
- let decoder = Decoder::from_tokio(reader).await?;
+ let decoder = Decoder::from_tokio(reader, None).await?;
extract_sub_dir_seq(&target, decoder).await?;
// we extracted a .pxarexclude-cli file auto-generated by the VM when encoding the
diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
index 2bbe90e34..d475da4e3 100644
--- a/pxar-bin/src/main.rs
+++ b/pxar-bin/src/main.rs
@@ -26,7 +26,7 @@ fn extract_archive_from_reader<R: std::io::Read>(
options: PxarExtractOptions,
) -> Result<(), Error> {
pbs_client::pxar::extract_archive(
- pxar::decoder::Decoder::from_std(reader)?,
+ pxar::decoder::Decoder::from_std(reader, None)?,
Path::new(target),
feature_flags,
|path| {
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 3ea174998..068b6a61e 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -1803,7 +1803,7 @@ pub fn pxar_file_download(
let archive_size = reader.archive_size();
let reader = LocalDynamicReadAt::new(reader);
- let decoder = Accessor::new(reader, archive_size).await?;
+ let decoder = Accessor::new(reader, archive_size, None).await?;
let root = decoder.open_root().await?;
let path = OsStr::from_bytes(file_path).to_os_string();
let file = root
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index 84557bce1..11fb2b4cd 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -1069,7 +1069,7 @@ fn restore_snapshots_to_tmpdir(
"File {file_num}: snapshot archive {source_datastore}:{snapshot}",
);
- let mut decoder = pxar::decoder::sync::Decoder::from_std(reader)?;
+ let mut decoder = pxar::decoder::sync::Decoder::from_std(reader, None)?;
let target_datastore = match store_map.target_store(&source_datastore) {
Some(datastore) => datastore,
@@ -1685,7 +1685,7 @@ fn restore_snapshot_archive<'a>(
reader: Box<dyn 'a + TapeRead>,
snapshot_path: &Path,
) -> Result<bool, Error> {
- let mut decoder = pxar::decoder::sync::Decoder::from_std(reader)?;
+ let mut decoder = pxar::decoder::sync::Decoder::from_std(reader, None)?;
match try_restore_snapshot_archive(worker, &mut decoder, snapshot_path) {
Ok(_) => Ok(true),
Err(err) => {
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index 5b68941a4..140c573c1 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -277,7 +277,7 @@ async fn open_dynamic_index(
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader: Arc<dyn ReadAt + Send + Sync> = Arc::new(LocalDynamicReadAt::new(reader));
- let accessor = Accessor::new(reader, archive_size).await?;
+ let accessor = Accessor::new(reader, archive_size, None).await?;
Ok((lookup_index, accessor))
}
diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs
index 252384b50..43d1cf9c3 100644
--- a/src/tape/file_formats/snapshot_archive.rs
+++ b/src/tape/file_formats/snapshot_archive.rs
@@ -59,7 +59,7 @@ pub fn tape_write_snapshot_archive<'a>(
}
let mut encoder =
- pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?;
+ pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata, None)?;
for filename in file_list.iter() {
let mut file = snapshot_reader.open_file(filename).map_err(|err| {
@@ -89,6 +89,7 @@ pub fn tape_write_snapshot_archive<'a>(
}
}
encoder.finish()?;
+ encoder.close()?;
Ok(())
});
--
2.39.2
More information about the pbs-devel
mailing list