[pbs-devel] [PATCH v8 proxmox-backup 65/69] client: pxar: add archive creation with reference test
Christian Ebner
c.ebner at proxmox.com
Tue May 28 11:42:59 CEST 2024
Add a basic regression test for archive creation with reference
metadata archive and index.
Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
changes since version 7:
- no changes
changes since version 6:
- adapt to PxarVariant pxar interface
- adapt to PxarLookaheadCache
pbs-client/src/pxar/create.rs | 243 ++++++++++++++++++
tests/pxar/backup-client-pxar-data.mpxar | Bin 0 -> 15070 bytes
tests/pxar/backup-client-pxar-data.ppxar.didx | Bin 0 -> 8096 bytes
tests/pxar/backup-client-pxar-expected.mpxar | Bin 0 -> 15086 bytes
4 files changed, 243 insertions(+)
create mode 100644 tests/pxar/backup-client-pxar-data.mpxar
create mode 100644 tests/pxar/backup-client-pxar-data.ppxar.didx
create mode 100644 tests/pxar/backup-client-pxar-expected.mpxar
diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index ff7e86804..bcadf12bd 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -1718,3 +1718,246 @@ fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec<u8> {
content
}
+
+#[cfg(test)]
+mod tests {
+ use std::ffi::OsString;
+ use std::fs::File;
+ use std::fs::OpenOptions;
+ use std::io::{self, BufReader, Seek, SeekFrom, Write};
+ use std::pin::Pin;
+ use std::process::Command;
+ use std::sync::mpsc;
+ use std::task::{Context, Poll};
+
+ use pbs_datastore::dynamic_index::DynamicIndexReader;
+ use pxar::accessor::sync::FileReader;
+ use pxar::encoder::SeqWrite;
+
+ use crate::pxar::extract::Extractor;
+ use crate::pxar::OverwriteFlags;
+
+ use super::*;
+
+ struct DummyWriter {
+ file: Option<File>,
+ }
+
+ impl DummyWriter {
+ fn new<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
+ let file = if let Some(path) = path {
+ Some(
+ OpenOptions::new()
+ .read(true)
+ .write(true)
+ .truncate(true)
+ .create(true)
+ .open(path)?,
+ )
+ } else {
+ None
+ };
+ Ok(Self { file })
+ }
+ }
+
+ impl Write for DummyWriter {
+ fn write(&mut self, data: &[u8]) -> io::Result<usize> {
+ if let Some(file) = self.file.as_mut() {
+ file.write_all(data)?;
+ }
+ Ok(data.len())
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ if let Some(file) = self.file.as_mut() {
+ file.flush()?;
+ }
+ Ok(())
+ }
+ }
+
+ impl SeqWrite for DummyWriter {
+ fn poll_seq_write(
+ mut self: Pin<&mut Self>,
+ _cx: &mut Context,
+ buf: &[u8],
+ ) -> Poll<io::Result<usize>> {
+ Poll::Ready(self.as_mut().write(buf))
+ }
+
+ fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<(), io::Error>> {
+ Poll::Ready(self.as_mut().flush())
+ }
+ }
+
+ fn prepare<P: AsRef<Path>>(dir_path: P) -> Result<(), Error> {
+ let dir = nix::dir::Dir::open(dir_path.as_ref(), OFlag::O_DIRECTORY, Mode::empty())?;
+
+ let fs_magic = detect_fs_type(dir.as_raw_fd()).unwrap();
+ let stat = nix::sys::stat::fstat(dir.as_raw_fd()).unwrap();
+ let mut fs_feature_flags = Flags::from_magic(fs_magic);
+ let metadata = get_metadata(
+ dir.as_raw_fd(),
+ &stat,
+ fs_feature_flags,
+ fs_magic,
+ &mut fs_feature_flags,
+ false,
+ )?;
+
+ let mut extractor = Extractor::new(
+ dir,
+ metadata.clone(),
+ true,
+ OverwriteFlags::empty(),
+ fs_feature_flags,
+ );
+
+ let dir_metadata = Metadata {
+ stat: pxar::Stat::default().mode(0o777u64).set_dir().gid(0).uid(0),
+ ..Default::default()
+ };
+
+ let file_metadata = Metadata {
+ stat: pxar::Stat::default()
+ .mode(0o777u64)
+ .set_regular_file()
+ .gid(0)
+ .uid(0),
+ ..Default::default()
+ };
+
+ extractor.enter_directory(
+ OsString::from(format!("testdir")),
+ dir_metadata.clone(),
+ true,
+ )?;
+
+ let size = 1024 * 1024;
+ let mut cursor = BufReader::new(std::io::Cursor::new(vec![0u8; size]));
+ for i in 0..10 {
+ extractor.enter_directory(
+ OsString::from(format!("folder_{i}")),
+ dir_metadata.clone(),
+ true,
+ )?;
+ for j in 0..10 {
+ cursor.seek(SeekFrom::Start(0))?;
+ extractor.extract_file(
+ CString::new(format!("file_{j}").as_str())?.as_c_str(),
+ &file_metadata,
+ size as u64,
+ &mut cursor,
+ true,
+ )?;
+ }
+ extractor.leave_directory()?;
+ }
+
+ extractor.leave_directory()?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_create_archive_with_reference() -> Result<(), Error> {
+ let mut testdir = PathBuf::from("./target/testout");
+ testdir.push(std::module_path!());
+
+ let _ = std::fs::remove_dir_all(&testdir);
+ let _ = std::fs::create_dir_all(&testdir);
+
+ prepare(testdir.as_path())?;
+
+ let previous_payload_index = Some(DynamicIndexReader::new(File::open(
+ "../tests/pxar/backup-client-pxar-data.ppxar.didx",
+ )?)?);
+ let metadata_archive = File::open("../tests/pxar/backup-client-pxar-data.mpxar").unwrap();
+ let metadata_size = metadata_archive.metadata()?.len();
+ let reader: MetadataArchiveReader = Arc::new(FileReader::new(metadata_archive));
+
+ let rt = tokio::runtime::Runtime::new().unwrap();
+ let (suggested_boundaries, _rx) = mpsc::channel();
+ let (forced_boundaries, _rx) = mpsc::channel();
+
+ rt.block_on(async move {
+ testdir.push("testdir");
+ let source_dir =
+ nix::dir::Dir::open(testdir.as_path(), OFlag::O_DIRECTORY, Mode::empty()).unwrap();
+
+ let fs_magic = detect_fs_type(source_dir.as_raw_fd()).unwrap();
+ let stat = nix::sys::stat::fstat(source_dir.as_raw_fd()).unwrap();
+ let mut fs_feature_flags = Flags::from_magic(fs_magic);
+
+ let metadata = get_metadata(
+ source_dir.as_raw_fd(),
+ &stat,
+ fs_feature_flags,
+ fs_magic,
+ &mut fs_feature_flags,
+ false,
+ )?;
+
+ let writer = DummyWriter::new(Some("./target/backup-client-pxar-run.mpxar")).unwrap();
+ let payload_writer = DummyWriter::new::<PathBuf>(None).unwrap();
+
+ let mut encoder = Encoder::new(
+ pxar::PxarVariant::Split(writer, payload_writer),
+ &metadata,
+ Some(&[]),
+ )
+ .await?;
+
+ let mut archiver = Archiver {
+ feature_flags: Flags::from_magic(fs_magic),
+ fs_feature_flags: Flags::from_magic(fs_magic),
+ fs_magic,
+ callback: Box::new(|_| Ok(())),
+ patterns: Vec::new(),
+ catalog: None,
+ path: PathBuf::new(),
+ entry_counter: 0,
+ entry_limit: 1024,
+ current_st_dev: stat.st_dev,
+ device_set: None,
+ hardlinks: HashMap::new(),
+ file_copy_buffer: vec::undefined(4 * 1024 * 1024),
+ skip_e2big_xattr: false,
+ forced_boundaries: Some(forced_boundaries),
+ previous_payload_index,
+ suggested_boundaries: Some(suggested_boundaries),
+ cache: PxarLookaheadCache::new(),
+ reuse_stats: ReuseStats::default(),
+ };
+
+ let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), metadata_size)
+ .await
+ .unwrap();
+ let root = accessor.open_root().await.ok();
+ archiver
+ .archive_dir_contents(&mut encoder, root, source_dir, true)
+ .await
+ .unwrap();
+
+ archiver
+ .flush_cached_reusing_if_below_threshold(&mut encoder, false)
+ .await
+ .unwrap();
+
+ encoder.finish().await.unwrap();
+ encoder.close().await.unwrap();
+
+ let status = Command::new("diff")
+ .args([
+ "../tests/pxar/backup-client-pxar-expected.mpxar",
+ "./target/backup-client-pxar-run.mpxar",
+ ])
+ .status()
+ .expect("failed to execute diff");
+ assert!(status.success());
+
+ Ok::<(), Error>(())
+ })
+ }
+}
diff --git a/tests/pxar/backup-client-pxar-data.mpxar b/tests/pxar/backup-client-pxar-data.mpxar
new file mode 100644
index 0000000000000000000000000000000000000000..00f3dc295fb38062c23e6cf7cac9ae110beb0a65
GIT binary patch
literal 15070
zcmeI3ZD<@t7{_Pd4&n>F7EIfqb#1^FO6}HK%_)tWO2s0r+iLp7VpnN`+SqK3 at uiTk
z0g)mo3n~RgSX7jP#Z`-;wUEWA!B4JW5kF|x5580 at T|bDmXzQ8GN at tzklRN*x`)~`#
z+|A9+Z+4!U-!r*zm%i41e0X5q&>}W-sk}V(=Du$q+4;h;F8=yl4}ZdoA2i1PeiW~F
z7gkDF&G*_D^Edhj2X^*7yu)JuwZnyZhYt+&$+{a8M{=R at jqX44;jVQr_n5qS`Ja!?
zJj=%~;8y>8^bO)nmIG_xu7%+&Cf=v??$*F?HnW6jmEx|0;T&euxV12x%N!baJq+hD
zm&V-y!}-jkaa}N6z<e54f#E_H2)HYTj;(+Fj+3hvDKpg*+uC;ZZa5IH;Qkxr`*hRW
zPwf8mu1mHb<?ZtNpKkkPvV3Urmo~1zy#C9{-_I`n@$iyOh4%etZrOKo^U622=`*~%
zeP!$T7i*WVKk;~>pO3D*w{v9smfybSqt4rptjHd`|MLm<Vqu)Wo?ABc{O-rP2Mg^x
z7k_iMt1TS)zR-W~W!=Mf|9sD>XZd*YdB}HcLEjPq_HYs}F67(1L&2w#Y%n&v?uz=3
zSjazEo-U<0$><xz#Vn$6IDIE9rg1oZr!1jyIDKa<rExfYGbN*OIDMBD#vM>&W#aU0
zDplb0RRf39x22dg4ySKhu>@R8-*xF*Vx%6v7kKeM>Dy6kA+B?*Z&z_>oMf`bW;a>I
z<m4$Xjl=2NS3DYr(|4fwG!CclPzh)pPT!Fd(m0&HV<n<-IDIEdOyh9+PL)K!we($=
zz9ow2nVpfOKE<8BGbI(`D#hVW-%QPD98TY5mGQr_Y8<H~u^F3PY>L^!RI9-0s|F6I
zZ%Z|498TZ1YSB2Hz8%%3aX5Xuszc*&`u0?p#^LnstDb;s>ANm{OZIGY=sQq-A+B?*
z?@$eB98TYn8qzqNzGF3_agwFbV75rqn8xAsovI0q!|6LyQyPcUH`6j2htqdiWBlvb
z8kruaZ&RxR&pTMO^j(*}C7Y-@^lfRT5Z5`@x2;(;4ySKNvuPYo->&A+IGnyc&82aY
zmDgal at HLOd;q)D7K8?faJJbRihtqeYg)|PQ?^ufjTua||>07d at n?v7;77KBmV|}Mu
zLgR4y&a{-q;q=Y)jK<;gUDg@@$9att98TY+UIm_af|D*4$wF^1TUfeD<8b=6b&JN~
z^zG<2jl=2N)g1xX(sy0 at mMpX8(6^_%LR_VL68GJ=uX{8Or|&@bX&g at 9p&rmUoW3JH
zq;WWX$9hELaQaU4n8r!=RfE|g)e{<r(|4w)G!Cb4W at G}crSH1*Es1+`=(}t%gFI5<
z^lchdAa#Pn>Dw|)8i&)jZCEr8r*FrwX&g at 9uHn!)oW4E7rExfY`-Vs3B-^;bY!Mhf
zjl=0XGy(zF(sy0 at mIR_X^c at +Y5Z5_AeaA*b<8b;;jF`sZ^qm?Bjl=0XGg2Cd(>E(+
zG!Ccla*375OpnvIS*il5g9T3CR>`Ds5^FS=E$osd;9B~Y>$^BF%I$l%bRK>#eY7&O
zHYWHE=v;8~)Sh?xU(H|VW$)?(9aB$LPP~7)*uLYlq5Vhi+jHX|?PC2`OI{f&Z9MeB
z>6K!Ahu7bI_2$0s#@C4T7d>;$s;8fP>(9z^v3}X<gBuoTx3=wFD%QVullIW at VSRMn
ee6jw{9WR|3pSSVg=*41v{&S{}`TgcUXZj0DmM>rc
literal 0
HcmV?d00001
diff --git a/tests/pxar/backup-client-pxar-data.ppxar.didx b/tests/pxar/backup-client-pxar-data.ppxar.didx
new file mode 100644
index 0000000000000000000000000000000000000000..a646218b5d504196443b17d62f3b22d171f011b8
GIT binary patch
literal 8096
zcmeIw&x=k`9LMqR`E|?Ga2Ha_;uI^9yBJNz!Z9f+>6R!pO*b*jh^~^|JnlRqSshV|
z%`G8TSEgiYa;C7OyU<J|9b=l(l;<{OBpc7nKj5>pIN$Ya@$KDb%dI01H%~o(*K{tQ
zJ~q6+^=PT==^y&xJ9`F3sC!@cUYwa2-S=W{^1`mZr(YJX_P=dkztmp8Zd>P0&yH!m
zYQlvAp+G1Q3WNfoKqwFjgaV;JC=d#S0-?bFT|iU3_TbPp*KU10`+DQ}SnEW!^~!_6
zmE|WN7T?dES$;Kh at A!s<^qNZtPc0p`(~IYOPkMW9;>Of`Ykp<tpHIL1?CtT1yY~$x
zkW0xxE~6B3Ic1P5D2JS-0&*o;$W>HA&QS%qnjGXj)sSn*LylMjxtI}Kh5y=%W?c!m
zglWhbmOw6L267ooA(yiZas|sFXITNcl3B=Atc09n736B>Am>>PxrTYj5pN(DbK=OZ
zH1A4ee_TV(@C0%xH;~JC3b~wTkSll&Im-*kmE1zE;w9u9uOL@*2RYAc$Ti$Ujzj~w
zSdc(=rA1dF`x6>+MkJ6+g at IfqQpn{ZgIpnU$XQW9t`rt>l_(+SL<PB8ILLWXL#`1X
zawHqb#gZhlD=oVc*`L&qGcti(Dh=c^nL;j?8RQC?L(a+qa;3D8t7Hi|Co9O+(m~G4
z8gh;FkR#PVE>@(FU1`;o$o`auoKXqnQe_~QsT6X#${<&$9CB6_kSmpiT%}6LIaNWf
zRt|Dr)sSnHha5!><l=}TWLG-sN at RbLhMb8K$YqgPb4Mq~o{fC&+#LA+d)TeK+0;Jx
Tczf at GpWkK|cE3#e4vqc=((xEu
literal 0
HcmV?d00001
diff --git a/tests/pxar/backup-client-pxar-expected.mpxar b/tests/pxar/backup-client-pxar-expected.mpxar
new file mode 100644
index 0000000000000000000000000000000000000000..ae4a18c89749f3d7ec82623e84509df19943d03e
GIT binary patch
literal 15086
zcmeI3Z-^9S9LJyew{ZQzQRvjGZ1W%mF~`ihExcw8BMEJ+&NoR;;T at Hij$M~!+%X3c
z5)=a!LLm(mg^)C*cv!*>U2*iP2{P$LIT8J_45t^7Nokw=%!_Ax+~4i?J=zyLa6C89
zJ@<T`XP)Qz{C>O3UiwDo@!`Q)L-SbmQh9m#&Zl18d#vMIli#0ud-r#bZF%Wv55GTG
z=D+abM~$(6erm4+b4!J*XM3IV`5y+h4{qsybhE|&Yln054j&rqmvuKLj^sk)8{PB%
zM_X6zEf;z7ykx98^L+dQZu!4Q-z3iBn7X*@U^tuQ^Q$wv6)>E`EdE&Q;I4<^TxQd_
zl`x#g92$264CgbK#@z_R1<a#yJuqCzd>U7R;UX3YxGRT_u72~*lgs8Q)#{0j9b5a>
z?2DIhA8zNZ*S-7XwomW5WYZDeF0cRj_D?3wgOk5 at a0TY|UrzpUcHvKl7p$vkKXB&O
z-6z*CeQTp$?Kp2=x at -K{%EhZsJW<on$5-9oJ+f)T?_cwA<n2e6WDh_1`2>5pW}LsB
zTQv3Jww=9syS(h4|IOK+j&S6Mn*RGP>m9!Lm-|jV&&QKLhg^R(`j!Z=%tywH3;8zh
zQ1GcF8jMY^yIOt6Ead-K$2gMFH;GGFMB{M!PFYOjaQe<zLgR4yW=cxqaQZftjK<;g
zT~ru%K%Je5)3>FVG!Cb4TdB<N{8eXmIDI>cCE(inZb;t}BbE7C;Kl!>Z&$H}b(Ka7
zoW4E9p>dLjH8#D6RU4dq#iemLeFut1<8b;86`#i8^c^Vyjl=0XRzezw(|4joG!Ccl
zREcREPT!f52)MSs8`8H#5#{L_N$OKv_RZ8(SXU_yr*BiuXdF)8MV0YaS#@$8$=Zxf
zZ*6L$g{7J_4ySKht<3NIRcCfMeLJc}<8b<RRh!1)^zEq*jl=2NS6v#1(|4eH0<Nv^
zhV(5tv#p`;Q1yj%ond`PYCz*~`i|9*#^Lmxs1c2mY=tJHMXJU$4yW%-O=uiW-%Lws
z98TY+meDwzzKa^;Z^zaNy*PbanknGg`ff<yl0(!Q`nI*oMxrt}T=wl~7LCK{+tq9u
zhts#GIW!KZZ(nn1oMh)U87%_Mqj5NWhni30aQcq4fX3nU9cv+t!|6NGA_3RdcSHJ?
z?CRFgcdEt0y3TO+ooNY;!|9vpDUHMF+tf1}htqdaXZ(ZnIvE^J-<EFDILStDGFsSr
zWqwp*fz!96TQm-*Z&$Zz98TY!?g+THz8liFWSg~yzJ1*l)^&#U9q1m7!|6NJeHw?;
zccce24yW%}4{01u--#a4IGnyyJ*IJzbJb+D$n=E9;q=Xnl*ZxoZ5o+?YwNoqeM{E8
zHS}FHm_g<^xHnHM!=!OIecMK5epCjB)3;+-G!Cb4*RW|EPT!v4&^VmFeZ!@3IDH3(
zN8=>NxXEY{8a|D~={qt40oT at dL;991L~H0fHbP-tXE^&#jEKhJ^qm?pjl=0XGZGqy
z(>E)mG!Cb4vyjm^oW6?%Rv<$!PTy9+q;WWX+l9*fsKi2IjV7aoQ?LYFTi<eh*FG2J
zj$IqN55JH;UaBtE1U~`Yb8ea1@!r7e`F&pYE#KEQ^-Sr+2Um#gyFMG*bL4>?H~rZu
z)_=9&wV}e=gCCw=D%N*-1HIR*@Be;$g;;;lbJs3=_UU*2DlHc47oFa}W{!4S$F7B9
o{h^z+M~)BcqpN0%^>=T6<;?i3wfjde7VGn`GkwA5n}40 at Z-J;Sk^lez
literal 0
HcmV?d00001
--
2.39.2
More information about the pbs-devel
mailing list