[pbs-devel] [RFC v2 pxar 07/36] encoder: add payload reference capability
Fabian Grünbichler
f.gruenbichler at proxmox.com
Mon Mar 11 14:21:59 CET 2024
On March 5, 2024 10:26 am, Christian Ebner wrote:
> Allows to encode regular files with a payload reference within a
> separate payload archive rather than encoding the payload within the
> regular archive.
>
> Following the PXAR_PAYLOAD_REF marked header, the payload offset and
> size are encoded.
>
> Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
> ---
> changes since version 1:
> - get rid of unused sub method impl for PayloadOffset
>
> src/encoder/aio.rs | 15 ++++++++++++++-
> src/encoder/mod.rs | 43 +++++++++++++++++++++++++++++++++++++++++++
> src/encoder/sync.rs | 18 +++++++++++++++++-
> 3 files changed, 74 insertions(+), 2 deletions(-)
>
> diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs
> index 60b11cd..87d75ef 100644
> --- a/src/encoder/aio.rs
> +++ b/src/encoder/aio.rs
> @@ -5,7 +5,7 @@ use std::path::Path;
> use std::pin::Pin;
> use std::task::{Context, Poll};
>
> -use crate::encoder::{self, LinkOffset, SeqWrite};
> +use crate::encoder::{self, LinkOffset, PayloadOffset, SeqWrite};
> use crate::format;
> use crate::Metadata;
>
> @@ -98,6 +98,19 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
> // content.as_async_reader(),
> // ).await
> // }
> + //
> + /// Encode a payload reference, returning the payload offset within the payload stream
> + pub async fn add_payload_ref(
> + &mut self,
> + metadata: &Metadata,
> + file_name: &Path,
> + file_size: u64,
> + payload_offset: PayloadOffset,
> + ) -> io::Result<()> {
> + self.inner
> + .add_payload_ref(metadata, file_name.as_ref(), file_size, payload_offset)
> + .await
> + }
>
> /// Create a new subdirectory. Note that the subdirectory has to be finished by calling the
> /// `finish()` method, otherwise the entire archive will be in an error state.
> diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
> index 962087a..1baa221 100644
> --- a/src/encoder/mod.rs
> +++ b/src/encoder/mod.rs
> @@ -38,6 +38,24 @@ impl LinkOffset {
> }
> }
>
> +/// File reference used to create payload references.
> +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd)]
> +pub struct PayloadOffset(u64);
> +
> +impl PayloadOffset {
> + /// Get the raw byte offset of this link.
> + #[inline]
> + pub fn raw(self) -> u64 {
> + self.0
> + }
> +
> + /// Return a new PayloadOffset, positively shifted by offset
> + #[inline]
> + pub fn add(&self, offset: u64) -> Self {
> + Self(self.0 + offset)
> + }
> +}
> +
> /// Sequential write interface used by the encoder's state machine.
> ///
> /// This is our internal writer trait which is available for `std::io::Write` types in the
> @@ -472,6 +490,31 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
> Ok(offset)
> }
>
> + /// Encode a payload reference, returning the payload offset within the payload stream
this doesn't return the offset though?
> + pub async fn add_payload_ref(
> + &mut self,
> + metadata: &Metadata,
> + file_name: &Path,
> + file_size: u64,
> + payload_offset: PayloadOffset,
> + ) -> io::Result<()> {
> + if self.payload_output.as_mut().is_none() {
> + io_bail!("unable to add payload reference");
> + }
> +
> + let mut payload_ref = payload_offset.raw().to_le_bytes().to_vec();
> + payload_ref.append(&mut file_size.to_le_bytes().to_vec());
another copy of manually encoding a PayloadRef (see patch #2)
> + let _this_offset: LinkOffset = self
> + .add_file_entry(
> + Some(metadata),
> + file_name,
> + Some((format::PXAR_PAYLOAD_REF, &payload_ref)),
> + )
> + .await?;
> +
> + Ok(())
> + }
> +
> /// Return a file offset usable with `add_hardlink`.
> pub async fn add_symlink(
> &mut self,
> diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs
> index a7c350e..90af8f9 100644
> --- a/src/encoder/sync.rs
> +++ b/src/encoder/sync.rs
> @@ -6,7 +6,7 @@ use std::pin::Pin;
> use std::task::{Context, Poll};
>
> use crate::decoder::sync::StandardReader;
> -use crate::encoder::{self, LinkOffset, SeqWrite};
> +use crate::encoder::{self, LinkOffset, PayloadOffset, SeqWrite};
> use crate::format;
> use crate::util::poll_result_once;
> use crate::Metadata;
> @@ -100,6 +100,22 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
> ))
> }
>
> + /// Encode a payload reference, returning the payload offset within the payload stream
> + pub async fn add_payload_ref(
> + &mut self,
> + metadata: &Metadata,
> + file_name: &Path,
> + file_size: u64,
> + payload_offset: PayloadOffset,
> + ) -> io::Result<()> {
> + poll_result_once(self.inner.add_payload_ref(
> + metadata,
> + file_name.as_ref(),
> + file_size,
> + payload_offset,
> + ))
> + }
> +
> /// Create a new subdirectory. Note that the subdirectory has to be finished by calling the
> /// `finish()` method, otherwise the entire archive will be in an error state.
> pub fn create_directory<P: AsRef<Path>>(
> --
> 2.39.2
>
>
>
> _______________________________________________
> pbs-devel mailing list
> pbs-devel at lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
>
>
>
More information about the pbs-devel
mailing list