[pbs-devel] [PATCH v6 proxmox-backup 61/65] client: pxar: add archive creation with reference test
Dominik Csapak
d.csapak at proxmox.com
Thu May 23 12:04:56 CEST 2024
interestingly this test runs through
but if do
./target/debug/pxar list ./tests/pxar/backup-client-pxar-expected.mpxar
i get the following error:
"/"
"/"
"/"
"/folder_0"
"/folder_0/file_0"
Error: unexpected entry type: UNKNOWN header (ef00315f656c6966)
is that expected ?
On 5/14/24 12:34, Christian Ebner wrote:
> Add a basic regression test for archive creation with reference
> metadata archive and index.
>
> Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
> ---
> pbs-client/src/pxar/create.rs | 242 ++++++++++++++++++
> tests/pxar/backup-client-pxar-data.mpxar | Bin 0 -> 15070 bytes
> tests/pxar/backup-client-pxar-data.ppxar.didx | Bin 0 -> 8096 bytes
> tests/pxar/backup-client-pxar-expected.mpxar | Bin 0 -> 15086 bytes
> 4 files changed, 242 insertions(+)
> create mode 100644 tests/pxar/backup-client-pxar-data.mpxar
> create mode 100644 tests/pxar/backup-client-pxar-data.ppxar.didx
> create mode 100644 tests/pxar/backup-client-pxar-expected.mpxar
>
> diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
> index 287e47655..9349dd8bb 100644
> --- a/pbs-client/src/pxar/create.rs
> +++ b/pbs-client/src/pxar/create.rs
> @@ -1811,3 +1811,245 @@ fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec<u8> {
>
> content
> }
> +
> +#[cfg(test)]
> +mod tests {
> + use std::ffi::OsString;
> + use std::fs::File;
> + use std::fs::OpenOptions;
> + use std::io::{self, BufReader, Seek, SeekFrom, Write};
> + use std::pin::Pin;
> + use std::process::Command;
> + use std::sync::mpsc;
> + use std::task::{Context, Poll};
> +
> + use pbs_datastore::dynamic_index::DynamicIndexReader;
> + use pxar::accessor::sync::FileReader;
> + use pxar::encoder::SeqWrite;
> +
> + use crate::pxar::extract::Extractor;
> + use crate::pxar::OverwriteFlags;
> +
> + use super::*;
> +
> + struct DummyWriter {
> + file: Option<File>,
> + }
> +
> + impl DummyWriter {
> + fn new<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
> + let file = if let Some(path) = path {
> + Some(
> + OpenOptions::new()
> + .read(true)
> + .write(true)
> + .truncate(true)
> + .create(true)
> + .open(path)?,
> + )
> + } else {
> + None
> + };
> + Ok(Self { file })
> + }
> + }
> +
> + impl Write for DummyWriter {
> + fn write(&mut self, data: &[u8]) -> io::Result<usize> {
> + if let Some(file) = self.file.as_mut() {
> + file.write_all(data)?;
> + }
> + Ok(data.len())
> + }
> +
> + fn flush(&mut self) -> io::Result<()> {
> + if let Some(file) = self.file.as_mut() {
> + file.flush()?;
> + }
> + Ok(())
> + }
> + }
> +
> + impl SeqWrite for DummyWriter {
> + fn poll_seq_write(
> + mut self: Pin<&mut Self>,
> + _cx: &mut Context,
> + buf: &[u8],
> + ) -> Poll<io::Result<usize>> {
> + Poll::Ready(self.as_mut().write(buf))
> + }
> +
> + fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<(), io::Error>> {
> + Poll::Ready(self.as_mut().flush())
> + }
> + }
> +
> + fn prepare<P: AsRef<Path>>(dir_path: P) -> Result<(), Error> {
> + let dir = nix::dir::Dir::open(dir_path.as_ref(), OFlag::O_DIRECTORY, Mode::empty())?;
> +
> + let fs_magic = detect_fs_type(dir.as_raw_fd()).unwrap();
> + let stat = nix::sys::stat::fstat(dir.as_raw_fd()).unwrap();
> + let mut fs_feature_flags = Flags::from_magic(fs_magic);
> + let metadata = get_metadata(
> + dir.as_raw_fd(),
> + &stat,
> + fs_feature_flags,
> + fs_magic,
> + &mut fs_feature_flags,
> + false,
> + )?;
> +
> + let mut extractor = Extractor::new(
> + dir,
> + metadata.clone(),
> + true,
> + OverwriteFlags::empty(),
> + fs_feature_flags,
> + );
> +
> + let dir_metadata = Metadata {
> + stat: pxar::Stat::default().mode(0o777u64).set_dir().gid(0).uid(0),
> + ..Default::default()
> + };
> +
> + let file_metadata = Metadata {
> + stat: pxar::Stat::default()
> + .mode(0o777u64)
> + .set_regular_file()
> + .gid(0)
> + .uid(0),
> + ..Default::default()
> + };
> +
> + extractor.enter_directory(
> + OsString::from(format!("testdir")),
> + dir_metadata.clone(),
> + true,
> + )?;
> +
> + let size = 1024 * 1024;
> + let mut cursor = BufReader::new(std::io::Cursor::new(vec![0u8; size]));
> + for i in 0..10 {
> + extractor.enter_directory(
> + OsString::from(format!("folder_{i}")),
> + dir_metadata.clone(),
> + true,
> + )?;
> + for j in 0..10 {
> + cursor.seek(SeekFrom::Start(0))?;
> + extractor.extract_file(
> + CString::new(format!("file_{j}").as_str())?.as_c_str(),
> + &file_metadata,
> + size as u64,
> + &mut cursor,
> + true,
> + )?;
> + }
> + extractor.leave_directory()?;
> + }
> +
> + extractor.leave_directory()?;
> +
> + Ok(())
> + }
> +
> + #[test]
> + fn test_create_archive_with_reference() -> Result<(), Error> {
> + let mut testdir = PathBuf::from("./target/testout");
> + testdir.push(std::module_path!());
> +
> + let _ = std::fs::remove_dir_all(&testdir);
> + let _ = std::fs::create_dir_all(&testdir);
> +
> + prepare(testdir.as_path())?;
> +
> + let previous_payload_index = Some(DynamicIndexReader::new(File::open(
> + "../tests/pxar/backup-client-pxar-data.ppxar.didx",
> + )?)?);
> + let metadata_archive = File::open("../tests/pxar/backup-client-pxar-data.mpxar").unwrap();
> + let metadata_size = metadata_archive.metadata()?.len();
> + let reader: MetadataArchiveReader = Arc::new(FileReader::new(metadata_archive));
> +
> + let rt = tokio::runtime::Runtime::new().unwrap();
> + let (suggested_boundaries, _rx) = mpsc::channel();
> + let (forced_boundaries, _rx) = mpsc::channel();
> +
> + rt.block_on(async move {
> + testdir.push("testdir");
> + let source_dir =
> + nix::dir::Dir::open(testdir.as_path(), OFlag::O_DIRECTORY, Mode::empty()).unwrap();
> +
> + let fs_magic = detect_fs_type(source_dir.as_raw_fd()).unwrap();
> + let stat = nix::sys::stat::fstat(source_dir.as_raw_fd()).unwrap();
> + let mut fs_feature_flags = Flags::from_magic(fs_magic);
> +
> + let metadata = get_metadata(
> + source_dir.as_raw_fd(),
> + &stat,
> + fs_feature_flags,
> + fs_magic,
> + &mut fs_feature_flags,
> + false,
> + )?;
> +
> + let mut writer =
> + DummyWriter::new(Some("./target/backup-client-pxar-run.mpxar")).unwrap();
> + let mut payload_writer = Some(DummyWriter::new::<PathBuf>(None).unwrap());
> +
> + let mut encoder =
> + Encoder::new(&mut writer, &metadata, payload_writer.as_mut(), Some(&[])).await?;
> +
> + let mut archiver = Archiver {
> + feature_flags: Flags::from_magic(fs_magic),
> + fs_feature_flags: Flags::from_magic(fs_magic),
> + fs_magic,
> + callback: Box::new(|_| Ok(())),
> + patterns: Vec::new(),
> + catalog: None,
> + path: PathBuf::new(),
> + entry_counter: 0,
> + entry_limit: 1024,
> + current_st_dev: stat.st_dev,
> + device_set: None,
> + hardlinks: HashMap::new(),
> + file_copy_buffer: vec::undefined(4 * 1024 * 1024),
> + skip_e2big_xattr: false,
> + forced_boundaries: Some(forced_boundaries),
> + previous_payload_index,
> + suggested_boundaries: Some(suggested_boundaries),
> + cached_entries: Vec::new(),
> + cached_range: Range::default(),
> + cached_last_chunk: None,
> + cached_hardlinks: HashSet::new(),
> + caching_enabled: false,
> + reuse_stats: ReuseStats::default(),
> + };
> +
> + let accessor = Accessor::new(reader, metadata_size, None).await.unwrap();
> + let root = accessor.open_root().await.ok();
> + archiver
> + .archive_dir_contents(&mut encoder, root, source_dir, true)
> + .await
> + .unwrap();
> +
> + archiver
> + .flush_cached_reusing_if_below_threshold(&mut encoder, false)
> + .await
> + .unwrap();
> +
> + encoder.finish().await.unwrap();
> + encoder.close().await.unwrap();
> +
> + let status = Command::new("diff")
> + .args([
> + "../tests/pxar/backup-client-pxar-expected.mpxar",
> + "./target/backup-client-pxar-run.mpxar",
> + ])
> + .status()
> + .expect("failed to execute diff");
> + assert!(status.success());
> +
> + Ok::<(), Error>(())
> + })
> + }
> +}
> diff --git a/tests/pxar/backup-client-pxar-data.mpxar b/tests/pxar/backup-client-pxar-data.mpxar
> new file mode 100644
> index 0000000000000000000000000000000000000000..00f3dc295fb38062c23e6cf7cac9ae110beb0a65
> GIT binary patch
> literal 15070
> zcmeI3ZD<@t7{_Pd4&n>F7EIfqb#1^FO6}HK%_)tWO2s0r+iLp7VpnN`+SqK3 at uiTk
> z0g)mo3n~RgSX7jP#Z`-;wUEWA!B4JW5kF|x5580 at T|bDmXzQ8GN at tzklRN*x`)~`#
> z+|A9+Z+4!U-!r*zm%i41e0X5q&>}W-sk}V(=Du$q+4;h;F8=yl4}ZdoA2i1PeiW~F
> z7gkDF&G*_D^Edhj2X^*7yu)JuwZnyZhYt+&$+{a8M{=R at jqX44;jVQr_n5qS`Ja!?
> zJj=%~;8y>8^bO)nmIG_xu7%+&Cf=v??$*F?HnW6jmEx|0;T&euxV12x%N!baJq+hD
> zm&V-y!}-jkaa}N6z<e54f#E_H2)HYTj;(+Fj+3hvDKpg*+uC;ZZa5IH;Qkxr`*hRW
> zPwf8mu1mHb<?ZtNpKkkPvV3Urmo~1zy#C9{-_I`n@$iyOh4%etZrOKo^U622=`*~%
> zeP!$T7i*WVKk;~>pO3D*w{v9smfybSqt4rptjHd`|MLm<Vqu)Wo?ABc{O-rP2Mg^x
> z7k_iMt1TS)zR-W~W!=Mf|9sD>XZd*YdB}HcLEjPq_HYs}F67(1L&2w#Y%n&v?uz=3
> zSjazEo-U<0$><xz#Vn$6IDIE9rg1oZr!1jyIDKa<rExfYGbN*OIDMBD#vM>&W#aU0
> zDplb0RRf39x22dg4ySKhu>@R8-*xF*Vx%6v7kKeM>Dy6kA+B?*Z&z_>oMf`bW;a>I
> z<m4$Xjl=2NS3DYr(|4fwG!CclPzh)pPT!Fd(m0&HV<n<-IDIEdOyh9+PL)K!we($=
> zz9ow2nVpfOKE<8BGbI(`D#hVW-%QPD98TY5mGQr_Y8<H~u^F3PY>L^!RI9-0s|F6I
> zZ%Z|498TZ1YSB2Hz8%%3aX5Xuszc*&`u0?p#^LnstDb;s>ANm{OZIGY=sQq-A+B?*
> z?@$eB98TYn8qzqNzGF3_agwFbV75rqn8xAsovI0q!|6LyQyPcUH`6j2htqdiWBlvb
> z8kruaZ&RxR&pTMO^j(*}C7Y-@^lfRT5Z5`@x2;(;4ySKNvuPYo->&A+IGnyc&82aY
> zmDgal at HLOd;q)D7K8?faJJbRihtqeYg)|PQ?^ufjTua||>07d at n?v7;77KBmV|}Mu
> zLgR4y&a{-q;q=Y)jK<;gUDg@@$9att98TY+UIm_af|D*4$wF^1TUfeD<8b=6b&JN~
> z^zG<2jl=2N)g1xX(sy0 at mMpX8(6^_%LR_VL68GJ=uX{8Or|&@bX&g at 9p&rmUoW3JH
> zq;WWX$9hELaQaU4n8r!=RfE|g)e{<r(|4w)G!Cb4W at G}crSH1*Es1+`=(}t%gFI5<
> z^lchdAa#Pn>Dw|)8i&)jZCEr8r*FrwX&g at 9uHn!)oW4E7rExfY`-Vs3B-^;bY!Mhf
> zjl=0XGy(zF(sy0 at mIR_X^c at +Y5Z5_AeaA*b<8b;;jF`sZ^qm?Bjl=0XGg2Cd(>E(+
> zG!Ccla*375OpnvIS*il5g9T3CR>`Ds5^FS=E$osd;9B~Y>$^BF%I$l%bRK>#eY7&O
> zHYWHE=v;8~)Sh?xU(H|VW$)?(9aB$LPP~7)*uLYlq5Vhi+jHX|?PC2`OI{f&Z9MeB
> z>6K!Ahu7bI_2$0s#@C4T7d>;$s;8fP>(9z^v3}X<gBuoTx3=wFD%QVullIW at VSRMn
> ee6jw{9WR|3pSSVg=*41v{&S{}`TgcUXZj0DmM>rc
>
> literal 0
> HcmV?d00001
>
> diff --git a/tests/pxar/backup-client-pxar-data.ppxar.didx b/tests/pxar/backup-client-pxar-data.ppxar.didx
> new file mode 100644
> index 0000000000000000000000000000000000000000..a646218b5d504196443b17d62f3b22d171f011b8
> GIT binary patch
> literal 8096
> zcmeIw&x=k`9LMqR`E|?Ga2Ha_;uI^9yBJNz!Z9f+>6R!pO*b*jh^~^|JnlRqSshV|
> z%`G8TSEgiYa;C7OyU<J|9b=l(l;<{OBpc7nKj5>pIN$Ya@$KDb%dI01H%~o(*K{tQ
> zJ~q6+^=PT==^y&xJ9`F3sC!@cUYwa2-S=W{^1`mZr(YJX_P=dkztmp8Zd>P0&yH!m
> zYQlvAp+G1Q3WNfoKqwFjgaV;JC=d#S0-?bFT|iU3_TbPp*KU10`+DQ}SnEW!^~!_6
> zmE|WN7T?dES$;Kh at A!s<^qNZtPc0p`(~IYOPkMW9;>Of`Ykp<tpHIL1?CtT1yY~$x
> zkW0xxE~6B3Ic1P5D2JS-0&*o;$W>HA&QS%qnjGXj)sSn*LylMjxtI}Kh5y=%W?c!m
> zglWhbmOw6L267ooA(yiZas|sFXITNcl3B=Atc09n736B>Am>>PxrTYj5pN(DbK=OZ
> zH1A4ee_TV(@C0%xH;~JC3b~wTkSll&Im-*kmE1zE;w9u9uOL@*2RYAc$Ti$Ujzj~w
> zSdc(=rA1dF`x6>+MkJ6+g at IfqQpn{ZgIpnU$XQW9t`rt>l_(+SL<PB8ILLWXL#`1X
> zawHqb#gZhlD=oVc*`L&qGcti(Dh=c^nL;j?8RQC?L(a+qa;3D8t7Hi|Co9O+(m~G4
> z8gh;FkR#PVE>@(FU1`;o$o`auoKXqnQe_~QsT6X#${<&$9CB6_kSmpiT%}6LIaNWf
> zRt|Dr)sSnHha5!><l=}TWLG-sN at RbLhMb8K$YqgPb4Mq~o{fC&+#LA+d)TeK+0;Jx
> Tczf at GpWkK|cE3#e4vqc=((xEu
>
> literal 0
> HcmV?d00001
>
> diff --git a/tests/pxar/backup-client-pxar-expected.mpxar b/tests/pxar/backup-client-pxar-expected.mpxar
> new file mode 100644
> index 0000000000000000000000000000000000000000..ae4a18c89749f3d7ec82623e84509df19943d03e
> GIT binary patch
> literal 15086
> zcmeI3Z-^9S9LJyew{ZQzQRvjGZ1W%mF~`ihExcw8BMEJ+&NoR;;T at Hij$M~!+%X3c
> z5)=a!LLm(mg^)C*cv!*>U2*iP2{P$LIT8J_45t^7Nokw=%!_Ax+~4i?J=zyLa6C89
> zJ@<T`XP)Qz{C>O3UiwDo@!`Q)L-SbmQh9m#&Zl18d#vMIli#0ud-r#bZF%Wv55GTG
> z=D+abM~$(6erm4+b4!J*XM3IV`5y+h4{qsybhE|&Yln054j&rqmvuKLj^sk)8{PB%
> zM_X6zEf;z7ykx98^L+dQZu!4Q-z3iBn7X*@U^tuQ^Q$wv6)>E`EdE&Q;I4<^TxQd_
> zl`x#g92$264CgbK#@z_R1<a#yJuqCzd>U7R;UX3YxGRT_u72~*lgs8Q)#{0j9b5a>
> z?2DIhA8zNZ*S-7XwomW5WYZDeF0cRj_D?3wgOk5 at a0TY|UrzpUcHvKl7p$vkKXB&O
> z-6z*CeQTp$?Kp2=x at -K{%EhZsJW<on$5-9oJ+f)T?_cwA<n2e6WDh_1`2>5pW}LsB
> zTQv3Jww=9syS(h4|IOK+j&S6Mn*RGP>m9!Lm-|jV&&QKLhg^R(`j!Z=%tywH3;8zh
> zQ1GcF8jMY^yIOt6Ead-K$2gMFH;GGFMB{M!PFYOjaQe<zLgR4yW=cxqaQZftjK<;g
> zT~ru%K%Je5)3>FVG!Cb4TdB<N{8eXmIDI>cCE(inZb;t}BbE7C;Kl!>Z&$H}b(Ka7
> zoW4E9p>dLjH8#D6RU4dq#iemLeFut1<8b;86`#i8^c^Vyjl=0XRzezw(|4joG!Ccl
> zREcREPT!f52)MSs8`8H#5#{L_N$OKv_RZ8(SXU_yr*BiuXdF)8MV0YaS#@$8$=Zxf
> zZ*6L$g{7J_4ySKht<3NIRcCfMeLJc}<8b<RRh!1)^zEq*jl=2NS6v#1(|4eH0<Nv^
> zhV(5tv#p`;Q1yj%ond`PYCz*~`i|9*#^Lmxs1c2mY=tJHMXJU$4yW%-O=uiW-%Lws
> z98TY+meDwzzKa^;Z^zaNy*PbanknGg`ff<yl0(!Q`nI*oMxrt}T=wl~7LCK{+tq9u
> zhts#GIW!KZZ(nn1oMh)U87%_Mqj5NWhni30aQcq4fX3nU9cv+t!|6NGA_3RdcSHJ?
> z?CRFgcdEt0y3TO+ooNY;!|9vpDUHMF+tf1}htqdaXZ(ZnIvE^J-<EFDILStDGFsSr
> zWqwp*fz!96TQm-*Z&$Zz98TY!?g+THz8liFWSg~yzJ1*l)^&#U9q1m7!|6NJeHw?;
> zccce24yW%}4{01u--#a4IGnyyJ*IJzbJb+D$n=E9;q=Xnl*ZxoZ5o+?YwNoqeM{E8
> zHS}FHm_g<^xHnHM!=!OIecMK5epCjB)3;+-G!Cb4*RW|EPT!v4&^VmFeZ!@3IDH3(
> zN8=>NxXEY{8a|D~={qt40oT at dL;991L~H0fHbP-tXE^&#jEKhJ^qm?pjl=0XGZGqy
> z(>E)mG!Cb4vyjm^oW6?%Rv<$!PTy9+q;WWX+l9*fsKi2IjV7aoQ?LYFTi<eh*FG2J
> zj$IqN55JH;UaBtE1U~`Yb8ea1@!r7e`F&pYE#KEQ^-Sr+2Um#gyFMG*bL4>?H~rZu
> z)_=9&wV}e=gCCw=D%N*-1HIR*@Be;$g;;;lbJs3=_UU*2DlHc47oFa}W{!4S$F7B9
> o{h^z+M~)BcqpN0%^>=T6<;?i3wfjde7VGn`GkwA5n}40 at Z-J;Sk^lez
>
> literal 0
> HcmV?d00001
>
More information about the pbs-devel
mailing list