[pbs-devel] [PATCH v6 proxmox-backup 13/29] fix #3174: archiver/extractor: impl appendix ref
Christian Ebner
c.ebner at proxmox.com
Thu Jan 25 14:25:52 CET 2024
Implements the functionality to create and extract appendix references
via the pbs client.
This reuses the pxar encoders functionality to write appendix reference
entries and adds the implementation to store and access them in the
catalog.
Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
Changes since v5:
- no changes
pbs-client/src/catalog_shell.rs | 2 +-
pbs-client/src/pxar/create.rs | 12 ++++
pbs-client/src/pxar/extract.rs | 16 +++++
pbs-client/src/pxar/tools.rs | 8 +++
pbs-datastore/src/catalog.rs | 103 ++++++++++++++++++++++++++++++++
5 files changed, 140 insertions(+), 1 deletion(-)
diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs
index 349bb7cb..8629a3a5 100644
--- a/pbs-client/src/catalog_shell.rs
+++ b/pbs-client/src/catalog_shell.rs
@@ -1085,7 +1085,7 @@ impl<'a> ExtractorState<'a> {
(_, DirEntryAttribute::Directory { .. }) => {
self.handle_new_directory(entry, match_result?).await?;
}
- (true, DirEntryAttribute::File { .. }) => {
+ (true, DirEntryAttribute::File { .. } | DirEntryAttribute::AppendixRef { .. }) => {
self.dir_stack.push(PathStackEntry::new(entry));
let file = Shell::walk_pxar_archive(self.accessor, &mut self.dir_stack).await?;
self.extract_file(file).await?;
diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index a2338218..611d7421 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -735,6 +735,18 @@ impl Archiver {
Ok(out.file_offset())
}
+ async fn add_appendix_ref<T: SeqWrite + Send>(
+ &mut self,
+ encoder: &mut Encoder<'_, T>,
+ file_name: &Path,
+ appendix_offset: pxar::encoder::AppendixRefOffset,
+ file_size: u64,
+ ) -> Result<(), Error> {
+ Ok(encoder
+ .add_appendix_ref(file_name, appendix_offset, file_size)
+ .await?)
+ }
+
async fn add_symlink<T: SeqWrite + Send>(
&mut self,
encoder: &mut Encoder<'_, T>,
diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index af18ecfc..f56c8f24 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -74,6 +74,7 @@ struct ExtractorIterState {
err_path_stack: Vec<OsString>,
current_match: bool,
end_reached: bool,
+ appendix_list: Vec<(PathBuf, u64, u64)>,
}
/// An [`Iterator`] that encapsulates the process of extraction in [extract_archive].
@@ -98,6 +99,7 @@ impl ExtractorIterState {
err_path_stack: Vec::new(),
current_match: options.extract_match_default,
end_reached: false,
+ appendix_list: Vec::new(),
}
}
}
@@ -373,6 +375,20 @@ where
}
.context(PxarExtractContext::ExtractFile)
}
+ (
+ true,
+ EntryKind::AppendixRef {
+ appendix_offset,
+ file_size,
+ },
+ ) => {
+ self.state.appendix_list.push((
+ entry.path().to_path_buf(),
+ *appendix_offset,
+ *file_size,
+ ));
+ Ok(())
+ }
(false, _) => Ok(()), // skip this
};
diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 0cfbaf5b..aac5a1e7 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -156,6 +156,14 @@ pub fn format_multi_line_entry(entry: &Entry) -> String {
let (size, link, type_name) = match entry.kind() {
EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"),
+ EntryKind::AppendixRef {
+ appendix_offset,
+ file_size,
+ } => (
+ format!("{} {}", appendix_offset, file_size),
+ String::new(),
+ "appendix ref",
+ ),
EntryKind::Symlink(link) => (
"0".to_string(),
format!(" -> {:?}", link.as_os_str()),
diff --git a/pbs-datastore/src/catalog.rs b/pbs-datastore/src/catalog.rs
index 746b493b..8ae7c661 100644
--- a/pbs-datastore/src/catalog.rs
+++ b/pbs-datastore/src/catalog.rs
@@ -28,6 +28,14 @@ pub trait BackupCatalogWriter {
ctime: i64,
file_offset: pxar::encoder::LinkOffset,
) -> Result<(), Error>;
+ fn add_appendix_ref(
+ &mut self,
+ name: &CStr,
+ size: u64,
+ mtime: i64,
+ ctime: i64,
+ appendix_ref_offset: pxar::encoder::AppendixRefOffset,
+ ) -> Result<(), Error>;
fn add_symlink(&mut self, name: &CStr) -> Result<(), Error>;
fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error>;
fn add_block_device(&mut self, name: &CStr) -> Result<(), Error>;
@@ -41,6 +49,7 @@ pub trait BackupCatalogWriter {
pub enum CatalogEntryType {
Directory = b'd',
File = b'f',
+ AppendixRef = b'r',
Symlink = b'l',
Hardlink = b'h',
BlockDevice = b'b',
@@ -56,6 +65,7 @@ impl TryFrom<u8> for CatalogEntryType {
Ok(match value {
b'd' => CatalogEntryType::Directory,
b'f' => CatalogEntryType::File,
+ b'r' => CatalogEntryType::AppendixRef,
b'l' => CatalogEntryType::Symlink,
b'h' => CatalogEntryType::Hardlink,
b'b' => CatalogEntryType::BlockDevice,
@@ -72,6 +82,7 @@ impl From<&DirEntryAttribute> for CatalogEntryType {
match value {
DirEntryAttribute::Directory { .. } => CatalogEntryType::Directory,
DirEntryAttribute::File { .. } => CatalogEntryType::File,
+ DirEntryAttribute::AppendixRef { .. } => CatalogEntryType::AppendixRef,
DirEntryAttribute::Symlink => CatalogEntryType::Symlink,
DirEntryAttribute::Hardlink => CatalogEntryType::Hardlink,
DirEntryAttribute::BlockDevice => CatalogEntryType::BlockDevice,
@@ -98,9 +109,22 @@ impl FileOffset {
self.offset
}
}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub struct AppendixRefOffset {
+ offset: u64,
+}
+
+impl AppendixRefOffset {
+ pub fn raw(&self) -> u64 {
+ self.offset
+ }
+}
+
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
pub enum Offset {
FileOffset { offset: u64 },
+ AppendixRefOffset { offset: u64 },
}
/// Represents a named directory entry
@@ -130,6 +154,12 @@ pub enum DirEntryAttribute {
mtime: i64,
extension: Option<CatalogV2Extension>,
},
+ AppendixRef {
+ size: u64,
+ mtime: i64,
+ ctime: i64,
+ appendix_ref_offset: AppendixRefOffset,
+ },
Symlink,
Hardlink,
BlockDevice,
@@ -195,6 +225,17 @@ impl DirEntry {
name,
attr: DirEntryAttribute::Socket,
},
+ (CatalogEntryType::AppendixRef, Some(Offset::AppendixRefOffset { offset })) => {
+ DirEntry {
+ name,
+ attr: DirEntryAttribute::AppendixRef {
+ size,
+ mtime,
+ ctime,
+ appendix_ref_offset: AppendixRefOffset { offset },
+ },
+ }
+ }
_ => panic!("unexpected parameters '{etype}' and '{offset:?}'"),
}
}
@@ -204,6 +245,7 @@ impl DirEntry {
Some(match self.attr {
DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
DirEntryAttribute::File { .. } => pxar::mode::IFREG,
+ DirEntryAttribute::AppendixRef { .. } => pxar::mode::IFREG,
DirEntryAttribute::Symlink => pxar::mode::IFLNK,
DirEntryAttribute::Hardlink => return None,
DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
@@ -271,6 +313,24 @@ impl DirInfo {
catalog_encode_u64(writer, file_offset.raw())?;
}
}
+ DirEntry {
+ name,
+ attr:
+ DirEntryAttribute::AppendixRef {
+ size,
+ mtime,
+ ctime,
+ appendix_ref_offset,
+ },
+ } => {
+ writer.write_all(&[CatalogEntryType::AppendixRef as u8])?;
+ catalog_encode_u64(writer, name.len() as u64)?;
+ writer.write_all(name)?;
+ catalog_encode_u64(writer, *size)?;
+ catalog_encode_i64(writer, *mtime)?;
+ catalog_encode_i64(writer, *ctime)?;
+ catalog_encode_u64(writer, appendix_ref_offset.raw())?;
+ }
DirEntry {
name,
attr: DirEntryAttribute::Symlink,
@@ -390,6 +450,21 @@ impl DirInfo {
};
callback(etype, name, 0, size, mtime, ctime, offset)?
}
+ CatalogEntryType::AppendixRef => {
+ let size = catalog_decode_u64(&mut cursor)?;
+ let mtime = catalog_decode_i64(&mut cursor)?;
+ let ctime = catalog_decode_i64(&mut cursor)?;
+ let offset = catalog_decode_u64(&mut cursor)?;
+ callback(
+ etype,
+ name,
+ 0,
+ size,
+ mtime,
+ ctime,
+ Some(Offset::AppendixRefOffset { offset }),
+ )?
+ }
_ => callback(etype, name, 0, 0, 0, 0, None)?,
};
if !cont {
@@ -517,6 +592,34 @@ impl<W: Write> BackupCatalogWriter for CatalogWriter<W> {
Ok(())
}
+ fn add_appendix_ref(
+ &mut self,
+ name: &CStr,
+ size: u64,
+ mtime: i64,
+ ctime: i64,
+ appendix_ref_offset: pxar::encoder::AppendixRefOffset,
+ ) -> Result<(), Error> {
+ let dir = self
+ .dirstack
+ .last_mut()
+ .ok_or_else(|| format_err!("outside root"))?;
+ let name = name.to_bytes().to_vec();
+ let appendix_ref_offset = AppendixRefOffset {
+ offset: appendix_ref_offset.raw(),
+ };
+ dir.entries.push(DirEntry {
+ name,
+ attr: DirEntryAttribute::AppendixRef {
+ size,
+ mtime,
+ ctime,
+ appendix_ref_offset,
+ },
+ });
+ Ok(())
+ }
+
fn add_symlink(&mut self, name: &CStr) -> Result<(), Error> {
let dir = self
.dirstack
--
2.39.2
More information about the pbs-devel
mailing list