[pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: add --dry-run parameter
Fabian Grünbichler
f.gruenbichler at proxmox.com
Fri Sep 16 12:35:33 CEST 2022
in dry-run mode, creating a snapshot will download (but not persist) the
Release files and any indices referenced within, but not download the
package files themselves. instead, any URLs that would still need to be
fetched are printed, and the statistics about to-be-fetched files and
bytes is updated accordingly.
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
src/bin/proxmox_offline_mirror_cmds/mirror.rs | 20 +++-
src/mirror.rs | 108 ++++++++++++++----
2 files changed, 102 insertions(+), 26 deletions(-)
diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
index 8059c0a..0798f86 100644
--- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
@@ -28,11 +28,22 @@ use super::get_config_path;
id: {
schema: MIRROR_ID_SCHEMA,
},
+ "dry-run": {
+ type: bool,
+ optional: true,
+ default: false,
+ description: "Only fetch indices and print summary of missing package files, don't store anything.",
+ }
},
},
)]
/// Create a new repository snapshot, fetching required/missing files from original repository.
-async fn create_snapshot(config: Option<String>, id: String, _param: Value) -> Result<(), Error> {
+async fn create_snapshot(
+ config: Option<String>,
+ id: String,
+ dry_run: bool,
+ _param: Value,
+) -> Result<(), Error> {
let config = config.unwrap_or_else(get_config_path);
let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
@@ -61,7 +72,12 @@ async fn create_snapshot(config: Option<String>, id: String, _param: Value) -> R
None
};
- proxmox_offline_mirror::mirror::create_snapshot(config, &Snapshot::now(), subscription)?;
+ proxmox_offline_mirror::mirror::create_snapshot(
+ config,
+ &Snapshot::now(),
+ subscription,
+ dry_run,
+ )?;
Ok(())
}
diff --git a/src/mirror.rs b/src/mirror.rs
index a1fc1a0..5126393 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -143,6 +143,7 @@ fn fetch_release(
config: &ParsedMirrorConfig,
prefix: &Path,
detached: bool,
+ dry_run: bool,
) -> Result<FetchResult, Error> {
let (name, fetched, sig) = if detached {
println!("Fetching Release/Release.gpg files");
@@ -185,6 +186,13 @@ fn fetch_release(
..Default::default()
};
+ if dry_run {
+ return Ok(FetchResult {
+ data: verified,
+ fetched: fetched.fetched,
+ });
+ }
+
let locked = &config.pool.lock()?;
if !locked.contains(&csums) {
@@ -234,6 +242,7 @@ fn fetch_index_file(
reference: &FileReference,
uncompressed: Option<&FileReference>,
by_hash: bool,
+ dry_run: bool,
) -> Result<FetchResult, Error> {
let url = get_dist_url(&config.repository, &reference.path);
let path = get_dist_path(&config.repository, prefix, &reference.path);
@@ -248,6 +257,9 @@ fn fetch_index_file(
.pool
.get_contents(&uncompressed.checksums, config.verify)?;
+ if dry_run {
+ return Ok(FetchResult { data, fetched: 0 });
+ }
// Ensure they're linked at current path
config.pool.lock()?.link_file(&reference.checksums, &path)?;
config
@@ -285,6 +297,7 @@ fn fetch_index_file(
reference.size,
&reference.checksums,
true,
+ dry_run,
)),
})
.ok_or_else(|| format_err!("Failed to retrieve {}", reference.path))??;
@@ -310,6 +323,14 @@ fn fetch_index_file(
&buf[..]
}
};
+ let res = FetchResult {
+ data: decompressed.to_owned(),
+ fetched: res.fetched,
+ };
+
+ if dry_run {
+ return Ok(res);
+ }
let locked = &config.pool.lock()?;
if let Some(uncompressed) = uncompressed {
@@ -322,10 +343,7 @@ fn fetch_index_file(
locked.link_file(&uncompressed.checksums, &uncompressed_path)?;
}
- Ok(FetchResult {
- data: decompressed.to_owned(),
- fetched: res.fetched,
- })
+ Ok(res)
}
/// Helper to fetch arbitrary files like binary packages.
@@ -340,6 +358,7 @@ fn fetch_plain_file(
max_size: usize,
checksums: &CheckSums,
need_data: bool,
+ dry_run: bool,
) -> Result<FetchResult, Error> {
let locked = &config.pool.lock()?;
let res = if locked.contains(checksums) {
@@ -355,6 +374,11 @@ fn fetch_plain_file(
fetched: 0,
}
}
+ } else if dry_run && !need_data {
+ FetchResult {
+ data: vec![],
+ fetched: 0,
+ }
} else {
let fetched = fetch_repo_file(
&config.client,
@@ -367,8 +391,10 @@ fn fetch_plain_file(
fetched
};
- // Ensure it's linked at current path
- locked.link_file(checksums, file)?;
+ if !dry_run {
+ // Ensure it's linked at current path
+ locked.link_file(checksums, file)?;
+ }
Ok(res)
}
@@ -433,6 +459,7 @@ pub fn create_snapshot(
config: MirrorConfig,
snapshot: &Snapshot,
subscription: Option<SubscriptionKey>,
+ dry_run: bool,
) -> Result<(), Error> {
let auth = if let Some(product) = &config.use_subscription {
match subscription {
@@ -477,11 +504,11 @@ pub fn create_snapshot(
};
// we want both on-disk for compat reasons
- let res = fetch_release(&config, prefix, true)?;
+ let res = fetch_release(&config, prefix, true, dry_run)?;
total_progress.update(&res);
let _release = parse_release(res, "Release")?;
- let res = fetch_release(&config, prefix, false)?;
+ let res = fetch_release(&config, prefix, false, dry_run)?;
total_progress.update(&res);
let release = parse_release(res, "InRelease")?;
@@ -591,6 +618,7 @@ pub fn create_snapshot(
reference,
uncompressed_ref,
release.aquire_by_hash,
+ dry_run,
) {
Ok(res) => res,
Err(err) if !reference.file_type.is_package_index() => {
@@ -632,6 +660,7 @@ pub fn create_snapshot(
}
println!("\nFetching packages..");
+ let mut dry_run_progress = Progress::new();
for (basename, references) in packages_indices {
let total_files = references.files.len();
if total_files == 0 {
@@ -643,30 +672,61 @@ pub fn create_snapshot(
let mut fetch_progress = Progress::new();
for package in references.files {
- let mut full_path = PathBuf::from(prefix);
- full_path.push(&package.file);
- let res = fetch_plain_file(
- &config,
- &get_repo_url(&config.repository, &package.file),
- &full_path,
- package.size,
- &package.checksums,
- false,
- )?;
- fetch_progress.update(&res);
+ let url = get_repo_url(&config.repository, &package.file);
+
+ if dry_run {
+ if config.pool.contains(&package.checksums) {
+ fetch_progress.update(&FetchResult {
+ data: vec![],
+ fetched: 0,
+ });
+ } else {
+ println!("\t(dry-run) GET missing '{url}' ({}b)", package.size);
+ fetch_progress.update(&FetchResult {
+ data: vec![],
+ fetched: package.size,
+ });
+ }
+ } else {
+ let mut full_path = PathBuf::from(prefix);
+ full_path.push(&package.file);
+
+ let res = fetch_plain_file(
+ &config,
+ &url,
+ &full_path,
+ package.size,
+ &package.checksums,
+ false,
+ dry_run,
+ )?;
+ fetch_progress.update(&res);
+ }
+
if fetch_progress.file_count() % (max(total_files / 100, 1)) == 0 {
println!("\tProgress: {fetch_progress}");
}
}
println!("\tProgress: {fetch_progress}");
- total_progress += fetch_progress;
+ if dry_run {
+ dry_run_progress += fetch_progress;
+ } else {
+ total_progress += fetch_progress;
+ }
}
- println!("\nStats: {total_progress}");
+ if dry_run {
+ println!("\nDry-run Stats (indices, downloaded but not persisted):\n{total_progress}");
+ println!("\nDry-run stats (packages, new == missing):\n{dry_run_progress}");
+ } else {
+ println!("\nStats: {total_progress}");
+ }
- println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
- let locked = config.pool.lock()?;
- locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
+ if !dry_run {
+ println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
+ let locked = config.pool.lock()?;
+ locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
+ }
Ok(())
}
--
2.30.2
More information about the pve-devel
mailing list