[pve-devel] [PATCH proxmox-offline-mirror 3/4] medium: add diff command
Fabian Grünbichler
f.gruenbichler at proxmox.com
Wed Sep 21 10:12:41 CEST 2022
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
src/bin/proxmox_offline_mirror_cmds/medium.rs | 107 +++++++++++++++++-
src/medium.rs | 103 ++++++++++++++++-
2 files changed, 207 insertions(+), 3 deletions(-)
diff --git a/src/bin/proxmox_offline_mirror_cmds/medium.rs b/src/bin/proxmox_offline_mirror_cmds/medium.rs
index b76e4e6..574f748 100644
--- a/src/bin/proxmox_offline_mirror_cmds/medium.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/medium.rs
@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
use anyhow::Error;
use serde_json::Value;
@@ -220,6 +220,108 @@ async fn sync(
Ok(Value::Null)
}
+#[api(
+ input: {
+ properties: {
+ config: {
+ type: String,
+ optional: true,
+ description: "Path to mirroring config file.",
+ },
+ id: {
+ schema: MEDIA_ID_SCHEMA,
+ },
+ verbose: {
+ type: bool,
+ optional: true,
+ default: false,
+ description: "Verbose output (print paths in addition to summary)."
+ },
+ }
+ },
+ )]
+/// Diff a medium
+async fn diff(
+ config: Option<String>,
+ id: String,
+ verbose: bool,
+ _param: Value,
+) -> Result<Value, Error> {
+ let config = config.unwrap_or_else(get_config_path);
+
+ let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
+ let config: MediaConfig = section_config.lookup("medium", &id)?;
+ let mut mirrors = Vec::with_capacity(config.mirrors.len());
+ for mirror in &config.mirrors {
+ let mirror: MirrorConfig = section_config.lookup("mirror", mirror)?;
+ mirrors.push(mirror);
+ }
+
+ let mut diffs = medium::diff(&config, mirrors)?;
+ let mut mirrors: Vec<String> = diffs.keys().cloned().collect();
+ mirrors.sort_unstable();
+
+ let sort_paths =
+ |(path, _): &(PathBuf, u64), (other_path, _): &(PathBuf, u64)| path.cmp(other_path);
+
+ let mut first = true;
+ for mirror in mirrors {
+ if first {
+ first = false;
+ } else {
+ println!();
+ }
+
+ println!("Mirror '{mirror}'");
+ if let Some(Some(mut diff)) = diffs.remove(&mirror) {
+ let mut total_size = 0;
+ println!("\t{} file(s) only on medium:", diff.added.paths.len());
+ if verbose {
+ diff.added.paths.sort_unstable_by(sort_paths);
+ diff.changed.paths.sort_unstable_by(sort_paths);
+ diff.removed.paths.sort_unstable_by(sort_paths);
+ }
+ for (path, size) in diff.added.paths {
+ if verbose {
+ println!("\t\t{path:?}: +{size}b");
+ }
+ total_size += size;
+ }
+ println!("\tTotal size: +{total_size}b");
+
+ total_size = 0;
+ println!(
+ "\n\t{} file(s) missing on medium:",
+ diff.removed.paths.len()
+ );
+ for (path, size) in diff.removed.paths {
+ if verbose {
+ println!("\t\t{path:?}: -{size}b");
+ }
+ total_size += size;
+ }
+ println!("\tTotal size: -{total_size}b");
+
+ total_size = 0;
+ println!(
+ "\n\t{} file(s) diff between source and medium:",
+ diff.changed.paths.len()
+ );
+ for (path, size) in diff.changed.paths {
+ if verbose {
+ println!("\t\t{path:?}: +-{size}b");
+ }
+ }
+ println!("\tSum of size differences: +-{total_size}b");
+ } else {
+ // TODO
+ println!("\tNot yet synced or no longer available on source side.");
+ }
+ }
+
+ Ok(Value::Null)
+}
+
pub fn medium_commands() -> CommandLineInterface {
let cmd_def = CliCommandMap::new()
.insert(
@@ -230,7 +332,8 @@ pub fn medium_commands() -> CommandLineInterface {
"status",
CliCommand::new(&API_METHOD_STATUS).arg_param(&["id"]),
)
- .insert("sync", CliCommand::new(&API_METHOD_SYNC).arg_param(&["id"]));
+ .insert("sync", CliCommand::new(&API_METHOD_SYNC).arg_param(&["id"]))
+ .insert("diff", CliCommand::new(&API_METHOD_DIFF).arg_param(&["id"]));
cmd_def.into()
}
diff --git a/src/medium.rs b/src/medium.rs
index bfd35b4..4093fc1 100644
--- a/src/medium.rs
+++ b/src/medium.rs
@@ -1,5 +1,7 @@
use std::{
collections::{HashMap, HashSet},
+ fs::Metadata,
+ os::linux::fs::MetadataExt,
path::{Path, PathBuf},
};
@@ -16,7 +18,7 @@ use crate::{
generate_repo_file_line,
mirror::pool,
pool::Pool,
- types::{Snapshot, SNAPSHOT_REGEX},
+ types::{Diff, Snapshot, SNAPSHOT_REGEX},
};
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
@@ -439,3 +441,102 @@ pub fn sync(
Ok(())
}
+
+/// Sync medium's content according to config.
+pub fn diff(
+ medium: &crate::config::MediaConfig,
+ mirrors: Vec<MirrorConfig>,
+) -> Result<HashMap<String, Option<Diff>>, Error> {
+ let medium_base = Path::new(&medium.mountpoint);
+ if !medium_base.exists() {
+ bail!("Medium mountpoint doesn't exist.");
+ }
+
+ let _lock = lock(medium_base)?;
+
+ let state =
+ load_state(medium_base)?.ok_or_else(|| format_err!("Medium not yet initializes."))?;
+
+ let mirror_state = get_mirror_state(medium, &state);
+
+ let pools: HashMap<String, String> =
+ state
+ .mirrors
+ .iter()
+ .fold(HashMap::new(), |mut map, (id, info)| {
+ map.insert(id.clone(), info.pool.clone());
+ map
+ });
+
+ let mut diffs = HashMap::new();
+
+ let convert_file_list_to_diff = |files: Vec<(PathBuf, Metadata)>, added: bool| -> Diff {
+ files
+ .into_iter()
+ .fold(Diff::default(), |mut diff, (file, meta)| {
+ if !meta.is_file() {
+ return diff;
+ }
+
+ let size = meta.st_size();
+ if added {
+ diff.added.paths.push((file, size));
+ } else {
+ diff.removed.paths.push((file, size));
+ }
+ diff
+ })
+ };
+
+ let get_target_pool =
+ |mirror_id: &str, mirror: Option<&MirrorConfig>| -> Result<Option<Pool>, Error> {
+ let mut mirror_base = medium_base.to_path_buf();
+ mirror_base.push(Path::new(mirror_id));
+
+ let mut mirror_pool = medium_base.to_path_buf();
+ let pool_dir = match pools.get(mirror_id) {
+ Some(pool_dir) => pool_dir.to_owned(),
+ None => {
+ if let Some(mirror) = mirror {
+ mirror_pool_dir(mirror)
+ } else {
+ return Ok(None);
+ }
+ }
+ };
+ mirror_pool.push(pool_dir);
+
+ Ok(Some(Pool::open(&mirror_base, &mirror_pool)?))
+ };
+
+ for mirror in mirrors.into_iter() {
+ let source_pool: Pool = pool(&mirror)?;
+
+ if !mirror_state.synced.contains(&mirror.id) {
+ let files = source_pool.lock()?.list_files()?;
+ diffs.insert(mirror.id, Some(convert_file_list_to_diff(files, false)));
+ continue;
+ }
+
+ let target_pool = get_target_pool(mirror.id.as_str(), Some(&mirror))?
+ .ok_or_else(|| format_err!("Failed to open target pool."))?;
+ diffs.insert(
+ mirror.id,
+ Some(source_pool.lock()?.diff_pools(&target_pool)?),
+ );
+ }
+
+ for dropped in mirror_state.target_only {
+ match get_target_pool(&dropped, None)? {
+ Some(pool) => {
+ let files = pool.lock()?.list_files()?;
+ diffs.insert(dropped, Some(convert_file_list_to_diff(files, false)));
+ }
+ None => {
+ diffs.insert(dropped, None);
+ }
+ }
+ }
+
+ Ok(diffs)
+}
--
2.30.2
More information about the pve-devel
mailing list