[pve-devel] [PATCH proxmox-offline-mirror 1/4] mirror: add exclusion of packages/sections
Fabian Grünbichler
f.gruenbichler at proxmox.com
Tue Oct 18 11:20:37 CEST 2022
to keep the size of mirror snapshots down by excluding unnecessary files
(e.g., games data, browsers, debug packages, ..).
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
Notes:
requires proxmox-apt with 'section' field
we could suggest excluding sections like 'games' in the
wizard/docs..
Cargo.toml | 1 +
debian/control | 2 +
src/bin/proxmox-offline-mirror.rs | 4 +-
src/bin/proxmox_offline_mirror_cmds/config.rs | 8 +++
src/config.rs | 40 ++++++++++++-
src/mirror.rs | 59 ++++++++++++++++++-
6 files changed, 111 insertions(+), 3 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index 76791c8..b2bb188 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ anyhow = "1.0"
base64 = "0.13"
bzip2 = "0.4"
flate2 = "1.0.22"
+globset = "0.4.8"
hex = "0.4.3"
lazy_static = "1.4"
nix = "0.24"
diff --git a/debian/control b/debian/control
index 0741a7b..9fe6605 100644
--- a/debian/control
+++ b/debian/control
@@ -10,6 +10,7 @@ Build-Depends: debhelper (>= 12),
librust-base64-0.13+default-dev,
librust-bzip2-0.4+default-dev,
librust-flate2-1+default-dev (>= 1.0.22-~~),
+ librust-globset-0.4+default-dev (>= 0.4.8-~~),
librust-hex-0.4+default-dev (>= 0.4.3-~~),
librust-lazy-static-1+default-dev (>= 1.4-~~),
librust-nix-0.24+default-dev,
@@ -57,6 +58,7 @@ Depends:
librust-base64-0.13+default-dev,
librust-bzip2-0.4+default-dev,
librust-flate2-1+default-dev (>= 1.0.22-~~),
+ librust-globset-0.4+default-dev (>= 0.4.8-~~),
librust-hex-0.4+default-dev (>= 0.4.3-~~),
librust-lazy-static-1+default-dev (>= 1.4-~~),
librust-nix-0.24+default-dev,
diff --git a/src/bin/proxmox-offline-mirror.rs b/src/bin/proxmox-offline-mirror.rs
index 522056b..07b6ce6 100644
--- a/src/bin/proxmox-offline-mirror.rs
+++ b/src/bin/proxmox-offline-mirror.rs
@@ -13,7 +13,7 @@ use proxmox_offline_mirror::helpers::tty::{
read_bool_from_tty, read_selection_from_tty, read_string_from_tty,
};
use proxmox_offline_mirror::{
- config::{save_config, MediaConfig, MirrorConfig},
+ config::{save_config, MediaConfig, MirrorConfig, SkipConfig},
mirror,
types::{ProductType, MEDIA_ID_SCHEMA, MIRROR_ID_SCHEMA},
};
@@ -387,6 +387,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Er
base_dir: base_dir.clone(),
use_subscription: None,
ignore_errors: false,
+ skip: SkipConfig::default(), // TODO sensible default?
});
}
}
@@ -401,6 +402,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Er
base_dir,
use_subscription,
ignore_errors: false,
+ skip: SkipConfig::default(),
};
configs.push(main_config);
diff --git a/src/bin/proxmox_offline_mirror_cmds/config.rs b/src/bin/proxmox_offline_mirror_cmds/config.rs
index 5ebf6d5..3ebf4ad 100644
--- a/src/bin/proxmox_offline_mirror_cmds/config.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/config.rs
@@ -266,6 +266,14 @@ pub fn update_mirror(
data.ignore_errors = ignore_errors
}
+ if let Some(skip_packages) = update.skip.skip_packages {
+ data.skip.skip_packages = Some(skip_packages);
+ }
+
+ if let Some(skip_sections) = update.skip.skip_sections {
+ data.skip.skip_sections = Some(skip_sections);
+ }
+
config.set_data(&id, "mirror", &data)?;
proxmox_offline_mirror::config::save_config(&config_file, &config)?;
diff --git a/src/config.rs b/src/config.rs
index be8f96b..39b1193 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -14,6 +14,38 @@ use crate::types::{
PROXMOX_SUBSCRIPTION_KEY_SCHEMA,
};
+/// Skip Configuration
+#[api(
+ properties: {
+ "skip-sections": {
+ type: Array,
+ optional: true,
+ items: {
+ type: String,
+ description: "Section name",
+ },
+ },
+ "skip-packages": {
+ type: Array,
+ optional: true,
+ items: {
+ type: String,
+ description: "Package name",
+ },
+ },
+ },
+)]
+#[derive(Default, Serialize, Deserialize, Updater, Clone, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct SkipConfig {
+ /// Sections which should be skipped
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub skip_sections: Option<Vec<String>>,
+ /// Packages which should be skipped, supports globbing
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub skip_packages: Option<Vec<String>>,
+}
+
#[api(
properties: {
id: {
@@ -46,6 +78,9 @@ use crate::types::{
optional: true,
default: false,
},
+ "skip": {
+ type: SkipConfig,
+ },
}
)]
#[derive(Clone, Debug, Serialize, Deserialize, Updater)]
@@ -73,6 +108,9 @@ pub struct MirrorConfig {
/// Whether to downgrade download errors to warnings
#[serde(default)]
pub ignore_errors: bool,
+ /// Skip package files using these criteria
+ #[serde(default, flatten)]
+ pub skip: SkipConfig,
}
#[api(
@@ -191,7 +229,7 @@ fn init() -> SectionConfig {
let mut config = SectionConfig::new(&MIRROR_ID_SCHEMA);
let mirror_schema = match MirrorConfig::API_SCHEMA {
- Schema::Object(ref obj_schema) => obj_schema,
+ Schema::AllOf(ref all_of_schema) => all_of_schema,
_ => unreachable!(),
};
let mirror_plugin = SectionConfigPlugin::new(
diff --git a/src/mirror.rs b/src/mirror.rs
index dfb4cc9..22dc716 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -7,12 +7,13 @@ use std::{
use anyhow::{bail, format_err, Error};
use flate2::bufread::GzDecoder;
+use globset::{Glob, GlobSetBuilder};
use nix::libc;
use proxmox_http::{client::sync::Client, HttpClient, HttpOptions};
use proxmox_sys::fs::file_get_contents;
use crate::{
- config::{MirrorConfig, SubscriptionKey},
+ config::{MirrorConfig, SkipConfig, SubscriptionKey},
convert_repo_line,
pool::Pool,
types::{Diff, Snapshot, SNAPSHOT_REGEX},
@@ -47,6 +48,7 @@ struct ParsedMirrorConfig {
pub auth: Option<String>,
pub client: Client,
pub ignore_errors: bool,
+ pub skip: SkipConfig,
}
impl TryInto<ParsedMirrorConfig> for MirrorConfig {
@@ -76,6 +78,7 @@ impl TryInto<ParsedMirrorConfig> for MirrorConfig {
auth: None,
client,
ignore_errors: self.ignore_errors,
+ skip: self.skip,
})
}
}
@@ -664,8 +667,22 @@ pub fn create_snapshot(
}
}
+ let skipped_package_globs = if let Some(skipped_packages) = &config.skip.skip_packages {
+ let mut globs = GlobSetBuilder::new();
+ for glob in skipped_packages {
+ let glob = Glob::new(glob)?;
+ globs.add(glob);
+ }
+ let globs = globs.build()?;
+ Some(globs)
+ } else {
+ None
+ };
+
println!("\nFetching packages..");
let mut dry_run_progress = Progress::new();
+ let mut total_skipped_count = 0usize;
+ let mut total_skipped_bytes = 0usize;
for (basename, references) in packages_indices {
let total_files = references.files.len();
if total_files == 0 {
@@ -676,7 +693,37 @@ pub fn create_snapshot(
}
let mut fetch_progress = Progress::new();
+ let mut skipped_count = 0usize;
+ let mut skipped_bytes = 0usize;
for package in references.files {
+ if let Some(ref sections) = &config.skip.skip_sections {
+ if sections.iter().any(|section| package.section == *section) {
+ println!(
+ "\tskipping {} - {}b (section '{}')",
+ package.package, package.size, package.section
+ );
+ skipped_count += 1;
+ skipped_bytes += package.size;
+ continue;
+ }
+ }
+ if let Some(skipped_package_globs) = &skipped_package_globs {
+ let matches = skipped_package_globs.matches(&package.package);
+ if !matches.is_empty() {
+ // safety, skipped_package_globs is set based on this
+ let globs = config.skip.skip_packages.as_ref().unwrap();
+ let matches: Vec<String> = matches.iter().map(|i| globs[*i].clone()).collect();
+ println!(
+ "\tskipping {} - {}b (package glob(s): {})",
+ package.package,
+ package.size,
+ matches.join(", ")
+ );
+ skipped_count += 1;
+ skipped_bytes += package.size;
+ continue;
+ }
+ }
let url = get_repo_url(&config.repository, &package.file);
if dry_run {
@@ -728,6 +775,11 @@ pub fn create_snapshot(
} else {
total_progress += fetch_progress;
}
+ if skipped_count > 0 {
+ total_skipped_count += skipped_count;
+ total_skipped_bytes += skipped_bytes;
+ println!("Skipped downloading {skipped_count} packages totalling {skipped_bytes}b");
+ }
}
if dry_run {
@@ -736,6 +788,11 @@ pub fn create_snapshot(
} else {
println!("\nStats: {total_progress}");
}
+ if total_count > 0 {
+ println!(
+ "Skipped downloading {total_skipped_count} packages totalling {total_skipped_bytes}b"
+ );
+ }
if !warnings.is_empty() {
eprintln!("Warnings:");
--
2.30.2
More information about the pve-devel
mailing list