[pbs-devel] [RFC v2 pxar 5/23] fix #3174: enc/dec: impl PXAR_APPENDIX entrytype
Christian Ebner
c.ebner at proxmox.com
Mon Oct 9 13:51:21 CEST 2023
Add an additional entry type for marking the start of a pxar archive
appendix section. The appendix is a concatenation of possibly
uncorrelated chunks, therefore not following the pxar archive format
anymore. The appendix is only used to access the file metadata and
payloads when a PXAR_APPENDIX_REF entry is encountered in the archive
before this point.
Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
Changes since v1:
- Use custom type for appendix start offset instead of raw `u64`
examples/mk-format-hashes.rs | 1 +
src/decoder/mod.rs | 9 +++++++++
src/encoder/aio.rs | 9 ++++++++-
src/encoder/mod.rs | 30 ++++++++++++++++++++++++++++++
src/encoder/sync.rs | 9 ++++++++-
src/format/mod.rs | 7 +++++++
src/lib.rs | 4 ++++
7 files changed, 67 insertions(+), 2 deletions(-)
diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs
index 8b4f5de..f068edd 100644
--- a/examples/mk-format-hashes.rs
+++ b/examples/mk-format-hashes.rs
@@ -12,6 +12,7 @@ const CONSTANTS: &[(&str, &str, &str)] = &[
"__PROXMOX_FORMAT_ENTRY__",
),
("", "PXAR_FILENAME", "__PROXMOX_FORMAT_FILENAME__"),
+ ("", "PXAR_APPENDIX", "__PROXMOX_FORMAT_APPENDIX__"),
("", "PXAR_SYMLINK", "__PROXMOX_FORMAT_SYMLINK__"),
("", "PXAR_DEVICE", "__PROXMOX_FORMAT_DEVICE__"),
("", "PXAR_XATTR", "__PROXMOX_FORMAT_XATTR__"),
diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs
index 70a8697..43a1122 100644
--- a/src/decoder/mod.rs
+++ b/src/decoder/mod.rs
@@ -295,6 +295,7 @@ impl<I: SeqRead> DecoderImpl<I> {
continue;
}
}
+ format::PXAR_APPENDIX => return Ok(Some(self.entry.take())),
_ => io_bail!(
"expected filename or directory-goodbye pxar entry, got: {}",
self.current_header,
@@ -546,6 +547,14 @@ impl<I: SeqRead> DecoderImpl<I> {
self.entry.kind = EntryKind::Device(self.read_device().await?);
return Ok(ItemResult::Entry);
}
+ format::PXAR_APPENDIX => {
+ let bytes = self.read_entry_as_bytes().await?;
+ let total = u64::from_le_bytes(bytes[0..8].try_into().unwrap());
+ self.entry.kind = EntryKind::Appendix {
+ total,
+ };
+ return Ok(ItemResult::Entry);
+ }
format::PXAR_PAYLOAD => {
let offset = seq_read_position(&mut self.input).await.transpose()?;
self.entry.kind = EntryKind::File {
diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs
index 66ea535..5c7c7b9 100644
--- a/src/encoder/aio.rs
+++ b/src/encoder/aio.rs
@@ -5,7 +5,7 @@ use std::path::Path;
use std::pin::Pin;
use std::task::{Context, Poll};
-use crate::encoder::{self, AppendixRefOffset, LinkOffset, SeqWrite};
+use crate::encoder::{self, AppendixRefOffset, AppendixStartOffset, LinkOffset, SeqWrite};
use crate::format;
use crate::Metadata;
@@ -124,6 +124,13 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
.await
}
+ /// Add the appendix start entry marker
+ ///
+ /// Returns the LinkOffset pointing after the entry, the appendix start offset
+ pub async fn add_appendix(&mut self, full_size: u64) -> io::Result<AppendixStartOffset> {
+ self.inner.add_appendix(full_size).await
+ }
+
/// Add a symbolic link to the archive.
pub async fn add_symlink<PF: AsRef<Path>, PT: AsRef<Path>>(
&mut self,
diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
index 78612a6..3270d52 100644
--- a/src/encoder/mod.rs
+++ b/src/encoder/mod.rs
@@ -65,6 +65,15 @@ impl AppendixRefOffset {
/// Offset pointing to the start of the appendix section of the archive.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
pub struct AppendixStartOffset(u64);
+
+impl AppendixStartOffset {
+ /// Get the raw byte start offset for this appenidx section.
+ #[inline]
+ pub fn raw(self) -> u64 {
+ self.0
+ }
+}
+
/// Sequential write interface used by the encoder's state machine.
///
/// This is our internal writer trait which is available for `std::io::Write` types in the
@@ -510,6 +519,27 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
Ok(())
}
+ /// Add the appendix start entry marker
+ ///
+ /// Returns the AppendixStartOffset pointing after the entry, the start of the appendix
+ /// section of the archive.
+ pub async fn add_appendix(&mut self, full_size: u64) -> io::Result<AppendixStartOffset> {
+ self.check()?;
+
+ let data = &full_size.to_le_bytes().to_vec();
+ seq_write_pxar_entry(
+ self.output.as_mut(),
+ format::PXAR_APPENDIX,
+ &data,
+ &mut self.state.write_position,
+ )
+ .await?;
+
+ let offset = self.position();
+
+ Ok(AppendixStartOffset(offset))
+ }
+
/// Return a file offset usable with `add_hardlink`.
pub async fn add_symlink(
&mut self,
diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs
index 2c9ea2b..2aaa6f2 100644
--- a/src/encoder/sync.rs
+++ b/src/encoder/sync.rs
@@ -6,7 +6,7 @@ use std::pin::Pin;
use std::task::{Context, Poll};
use crate::decoder::sync::StandardReader;
-use crate::encoder::{self, AppendixRefOffset, LinkOffset, SeqSink, SeqWrite};
+use crate::encoder::{self, AppendixRefOffset, AppendixStartOffset, LinkOffset, SeqSink, SeqWrite};
use crate::format;
use crate::util::poll_result_once;
use crate::Metadata;
@@ -124,6 +124,13 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
))
}
+ /// Add the appendix start entry marker
+ ///
+ /// Returns the LinkOffset pointing after the entry, the appendix start offset
+ pub async fn add_appendix(&mut self, full_size: u64) -> io::Result<AppendixStartOffset> {
+ poll_result_once(self.inner.add_appendix(full_size))
+ }
+
/// Add a symbolic link to the archive.
pub fn add_symlink<PF: AsRef<Path>, PT: AsRef<Path>>(
&mut self,
diff --git a/src/format/mod.rs b/src/format/mod.rs
index 5eb7562..8254df9 100644
--- a/src/format/mod.rs
+++ b/src/format/mod.rs
@@ -35,6 +35,12 @@
//! * `<archive>` -- serialization of the second directory entry
//! * ...
//! * `GOODBYE` -- lookup table at the end of a list of directory entries
+//!
+//! For backups referencing previous backups to skip file payloads, the archive is followed by a
+//! appendix maker after which the concatinated pxar archive fragments containing the file payloads
+//! are appended. They are NOT guaranteed to follow the full pxar structure and should only be
+//! used to extract the file payloads by given offset.
+//! * `APPENDIX` -- pxar archive fragments containing file payloads
use std::cmp::Ordering;
use std::ffi::{CStr, OsStr};
@@ -85,6 +91,7 @@ pub const PXAR_ENTRY: u64 = 0xd5956474e588acef;
/// Previous version of the entry struct
pub const PXAR_ENTRY_V1: u64 = 0x11da850a1c1cceff;
pub const PXAR_FILENAME: u64 = 0x16701121063917b3;
+pub const PXAR_APPENDIX: u64 = 0x9ff6c9507864b38d;
pub const PXAR_SYMLINK: u64 = 0x27f971e7dbf5dc5f;
pub const PXAR_DEVICE: u64 = 0x9fc9e906586d5ce9;
pub const PXAR_XATTR: u64 = 0x0dab0229b57dcd03;
diff --git a/src/lib.rs b/src/lib.rs
index fa84e7a..035f995 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -372,6 +372,10 @@ pub enum EntryKind {
file_size: u64,
},
+ Appendix {
+ total: u64,
+ },
+
/// Directory entry. When iterating through an archive, the contents follow next.
Directory,
--
2.39.2
More information about the pbs-devel
mailing list