[pve-devel] [PATCH v2 proxmox-backup 05/10] pull: allow pulling groups selectively
Fabian Grünbichler
f.gruenbichler at proxmox.com
Wed Sep 15 15:41:52 CEST 2021
without requiring workarounds based on ownership and limited
visibility/access.
if a group filter is set, remove_vanished will only consider filtered
groups for removal to prevent concurrent disjunct filters from trashing
eachother's synced groups.
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
other name options:
- `group-filter` (with `exclude` boolean as extension?)
- `include-groups` (with `exclude-groups` filter list as extension?)
src/api2/pull.rs | 13 ++++++++++---
src/bin/proxmox-backup-manager.rs | 10 ++++++++++
src/server/pull.rs | 32 ++++++++++++++++++++++++++++---
3 files changed, 49 insertions(+), 6 deletions(-)
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index d7b155a1..dfa6a938 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
use pbs_client::{HttpClient, BackupRepository};
use pbs_api_types::{
- Remote, Authid, SyncJobConfig,
+ Remote, Authid, SyncJobConfig, GroupFilterList,
DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ,
};
@@ -102,7 +102,7 @@ pub fn do_sync_job(
worker.log(format!("Sync datastore '{}' from '{}/{}'",
sync_job.store, sync_job.remote, sync_job.remote_store));
- pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner).await?;
+ pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner, None).await?;
worker.log(format!("sync job '{}' end", &job_id));
@@ -153,6 +153,10 @@ pub fn do_sync_job(
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
optional: true,
},
+ "groups": {
+ type: GroupFilterList,
+ optional: true,
+ },
},
},
access: {
@@ -170,6 +174,7 @@ async fn pull (
remote: String,
remote_store: String,
remove_vanished: Option<bool>,
+ groups: Option<GroupFilterList>,
_info: &ApiMethod,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
@@ -177,6 +182,8 @@ async fn pull (
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
let delete = remove_vanished.unwrap_or(true);
+ let groups = groups.map(GroupFilterList::filters);
+
check_pull_privs(&auth_id, &store, &remote, &remote_store, delete)?;
let (client, src_repo, tgt_store) = get_pull_parameters(&store, &remote, &remote_store).await?;
@@ -186,7 +193,7 @@ async fn pull (
worker.log(format!("sync datastore '{}' start", store));
- let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id);
+ let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id, groups);
let future = select!{
success = pull_future.fuse() => success,
abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort,
diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 689f44db..479ecf03 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -12,6 +12,7 @@ use pbs_tools::json::required_string_param;
use pbs_api_types::{
DATASTORE_SCHEMA, UPID_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
IGNORE_VERIFIED_BACKUPS_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA,
+ GroupFilterList,
};
use proxmox_backup::config;
@@ -234,6 +235,10 @@ fn task_mgmt_cli() -> CommandLineInterface {
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
optional: true,
},
+ "groups": {
+ type: GroupFilterList,
+ optional: true,
+ },
"output-format": {
schema: OUTPUT_FORMAT,
optional: true,
@@ -247,6 +252,7 @@ async fn pull_datastore(
remote_store: String,
local_store: String,
remove_vanished: Option<bool>,
+ groups: Option<GroupFilterList>,
param: Value,
) -> Result<Value, Error> {
@@ -260,6 +266,10 @@ async fn pull_datastore(
"remote-store": remote_store,
});
+ if groups.is_some() {
+ args["groups"] = json!(groups);
+ }
+
if let Some(remove_vanished) = remove_vanished {
args["remove-vanished"] = Value::from(remove_vanished);
}
diff --git a/src/server/pull.rs b/src/server/pull.rs
index 4e82df1a..2910d280 100644
--- a/src/server/pull.rs
+++ b/src/server/pull.rs
@@ -12,7 +12,7 @@ use serde_json::json;
use proxmox::api::error::{HttpError, StatusCode};
-use pbs_api_types::{Authid, SnapshotListItem, GroupListItem};
+use pbs_api_types::{Authid, SnapshotListItem, GroupListItem, GroupFilter};
use pbs_datastore::{task_log, BackupInfo, BackupDir, BackupGroup, StoreProgress};
use pbs_datastore::data_blob::DataBlob;
use pbs_datastore::dynamic_index::DynamicIndexReader;
@@ -631,6 +631,7 @@ pub async fn pull_store(
tgt_store: Arc<DataStore>,
delete: bool,
auth_id: Authid,
+ group_filter: Option<Vec<GroupFilter>>,
) -> Result<(), Error> {
// explicit create shared lock to prevent GC on newly created chunks
let _shared_store_lock = tgt_store.try_shared_chunk_store_lock()?;
@@ -644,8 +645,7 @@ pub async fn pull_store(
let mut list: Vec<GroupListItem> = serde_json::from_value(result["data"].take())?;
- worker.log(format!("found {} groups to sync", list.len()));
-
+ let total_count = list.len();
list.sort_unstable_by(|a, b| {
let type_order = a.backup_type.cmp(&b.backup_type);
if type_order == std::cmp::Ordering::Equal {
@@ -655,11 +655,32 @@ pub async fn pull_store(
}
});
+ let apply_filters = |group: &BackupGroup, filters: &[GroupFilter]| -> bool {
+ filters
+ .iter()
+ .any(|filter| group.filter(filter))
+ };
+
let list:Vec<BackupGroup> = list
.into_iter()
.map(|item| BackupGroup::new(item.backup_type, item.backup_id))
.collect();
+ let list = if let Some(ref group_filter) = &group_filter {
+ let unfiltered_count = list.len();
+ let list:Vec<BackupGroup> = list
+ .into_iter()
+ .filter(|group| {
+ apply_filters(&group, group_filter)
+ })
+ .collect();
+ worker.log(format!("found {} groups to sync (out of {} total)", list.len(), unfiltered_count));
+ list
+ } else {
+ worker.log(format!("found {} groups to sync", total_count));
+ list
+ };
+
let mut errors = false;
let mut new_groups = std::collections::HashSet::new();
@@ -720,6 +741,11 @@ pub async fn pull_store(
if new_groups.contains(&local_group) {
continue;
}
+ if let Some(ref group_filter) = &group_filter {
+ if !apply_filters(&local_group, group_filter) {
+ continue;
+ }
+ }
worker.log(format!(
"delete vanished group '{}/{}'",
local_group.backup_type(),
--
2.30.2
More information about the pve-devel
mailing list