[pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: also filter {component}/{section}

[Fabian Grünbichler]

so that a single filter 'games' covers all related sections for a stock
Debian repository.

this also has the side-effect that package downloads are now batched by
component.

Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
 src/mirror.rs | 58 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 40 insertions(+), 18 deletions(-)

diff --git a/src/mirror.rs b/src/mirror.rs
index e6aca8a..ea593a9 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -504,6 +504,7 @@ fn convert_to_globset(config: &ParsedMirrorConfig) -> Result<Option<GlobSet>, Er
 
 fn fetch_binary_packages(
     config: &ParsedMirrorConfig,
+    component: &str,
     packages_indices: HashMap<&String, PackagesFile>,
     dry_run: bool,
     prefix: &Path,
@@ -523,9 +524,13 @@ fn fetch_binary_packages(
         let mut fetch_progress = Progress::new();
         let mut skip_count = 0usize;
         let mut skip_bytes = 0usize;
+
         for package in references.files {
             if let Some(ref sections) = &config.skip.skip_sections {
-                if sections.iter().any(|section| package.section == *section) {
+                if sections.iter().any(|section| {
+                    package.section == *section
+                        || package.section == format!("{component}/{section}")
+                }) {
                     println!(
                         "\tskipping {} - {}b (section '{}')",
                         package.package, package.size, package.section
@@ -615,6 +620,7 @@ fn fetch_binary_packages(
 
 fn fetch_source_packages(
     config: &ParsedMirrorConfig,
+    component: &str,
     source_packages_indices: HashMap<&String, SourcesFile>,
     dry_run: bool,
     prefix: &Path,
@@ -636,10 +642,10 @@ fn fetch_source_packages(
         let mut skip_bytes = 0usize;
         for package in references.source_packages {
             if let Some(ref sections) = &config.skip.skip_sections {
-                if sections
-                    .iter()
-                    .any(|section| package.section.as_ref() == Some(section))
-                {
+                if sections.iter().any(|section| {
+                    package.section.as_ref() == Some(section)
+                        || package.section == Some(format!("{component}/{section}"))
+                }) {
                     println!(
                         "\tskipping {} - {}b (section '{}')",
                         package.package,
@@ -897,9 +903,13 @@ pub fn create_snapshot(
     println!();
 
     let mut packages_size = 0_usize;
-    let mut packages_indices = HashMap::new();
-
-    let mut source_packages_indices = HashMap::new();
+    let mut per_component_indices: HashMap<
+        String,
+        (
+            HashMap<&String, PackagesFile>,
+            HashMap<&String, SourcesFile>,
+        ),
+    > = HashMap::new();
 
     let mut failed_references = Vec::new();
     for (component, references) in per_component {
@@ -909,6 +919,9 @@ pub fn create_snapshot(
 
         let mut fetch_progress = Progress::new();
 
+        let (packages_indices, source_packages_indices) =
+            per_component_indices.entry(component.clone()).or_default();
+
         for basename in references {
             println!("\tFetching '{basename}'..");
             let files = release.files.get(basename).unwrap();
@@ -1001,17 +1014,26 @@ pub fn create_snapshot(
         }
     }
 
-    println!("\nFetching packages..");
-
-    fetch_binary_packages(&config, packages_indices, dry_run, prefix, &mut progress)?;
+    for (component, (packages_indices, source_packages_indices)) in per_component_indices {
+        println!("\nFetching {component} packages..");
+        fetch_binary_packages(
+            &config,
+            &component,
+            packages_indices,
+            dry_run,
+            prefix,
+            &mut progress,
+        )?;
 
-    fetch_source_packages(
-        &config,
-        source_packages_indices,
-        dry_run,
-        prefix,
-        &mut progress,
-    )?;
+        fetch_source_packages(
+            &config,
+            &component,
+            source_packages_indices,
+            dry_run,
+            prefix,
+            &mut progress,
+        )?;
+    }
 
     if dry_run {
         println!(
-- 
2.30.2