[pbs-devel] [RFC v2 proxmox-backup 18/42] api: backup: store datastore backend in runtime environment
Christian Ebner
c.ebner at proxmox.com
Thu May 29 16:31:43 CEST 2025
Get and store the datastore's backend during creation of the backup
runtime environment and upload the chunks to the local filesystem or
s3 object store based on the backend variant.
By storing the backend variant in the environment the s3 client is
instantiated only once and reused for all api calls in the same
backup http/2 connection.
Refactor the upgrade method by moving all logic into the async block,
such that the now possible error on backup environment creation gets
propagated to the thread spawn call side.
Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
src/api2/backup/environment.rs | 12 +++--
src/api2/backup/mod.rs | 99 +++++++++++++++++-----------------
2 files changed, 57 insertions(+), 54 deletions(-)
diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 6cd29f512..8919b919a 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -15,7 +15,8 @@ use pbs_api_types::Authid;
use pbs_datastore::backup_info::{BackupDir, BackupInfo};
use pbs_datastore::dynamic_index::DynamicIndexWriter;
use pbs_datastore::fixed_index::FixedIndexWriter;
-use pbs_datastore::{DataBlob, DataStore};
+use pbs_datastore::{DataBlob, DataStore, DatastoreBackend};
+use pbs_s3_client::PutObjectResponse;
use proxmox_rest_server::{formatter::*, WorkerTask};
use crate::backup::VerifyWorker;
@@ -115,6 +116,7 @@ pub struct BackupEnvironment {
pub datastore: Arc<DataStore>,
pub backup_dir: BackupDir,
pub last_backup: Option<BackupInfo>,
+ pub backend: DatastoreBackend,
state: Arc<Mutex<SharedBackupState>>,
}
@@ -125,7 +127,7 @@ impl BackupEnvironment {
worker: Arc<WorkerTask>,
datastore: Arc<DataStore>,
backup_dir: BackupDir,
- ) -> Self {
+ ) -> Result<Self, Error> {
let state = SharedBackupState {
finished: false,
uid_counter: 0,
@@ -137,7 +139,8 @@ impl BackupEnvironment {
backup_stat: UploadStatistic::new(),
};
- Self {
+ let backend = datastore.backend()?;
+ Ok(Self {
result_attributes: json!({}),
env_type,
auth_id,
@@ -147,8 +150,9 @@ impl BackupEnvironment {
formatter: JSON_FORMATTER,
backup_dir,
last_backup: None,
+ backend,
state: Arc::new(Mutex::new(state)),
- }
+ })
}
/// Register a Chunk with associated length.
diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs
index 567bca3ef..2c6afca41 100644
--- a/src/api2/backup/mod.rs
+++ b/src/api2/backup/mod.rs
@@ -185,7 +185,8 @@ fn upgrade_to_backup_protocol(
}
// lock last snapshot to prevent forgetting/pruning it during backup
- let guard = last.backup_dir
+ let guard = last
+ .backup_dir
.lock_shared()
.with_context(|| format!("while locking last snapshot during backup '{last:?}'"))?;
Some(guard)
@@ -204,14 +205,14 @@ fn upgrade_to_backup_protocol(
Some(worker_id),
auth_id.to_string(),
true,
- move |worker| {
+ move |worker| async move {
let mut env = BackupEnvironment::new(
env_type,
auth_id,
worker.clone(),
datastore,
backup_dir,
- );
+ )?;
env.debug = debug;
env.last_backup = last_backup;
@@ -264,55 +265,53 @@ fn upgrade_to_backup_protocol(
});
let mut abort_future = abort_future.map(|_| Err(format_err!("task aborted")));
- async move {
- // keep flock until task ends
- let _group_guard = _group_guard;
- let snap_guard = snap_guard;
- let _last_guard = _last_guard;
-
- let res = select! {
- req = req_fut => req,
- abrt = abort_future => abrt,
- };
- if benchmark {
- env.log("benchmark finished successfully");
- proxmox_async::runtime::block_in_place(|| env.remove_backup())?;
- return Ok(());
+ // keep flock until task ends
+ let _group_guard = _group_guard;
+ let snap_guard = snap_guard;
+ let _last_guard = _last_guard;
+
+ let res = select! {
+ req = req_fut => req,
+ abrt = abort_future => abrt,
+ };
+ if benchmark {
+ env.log("benchmark finished successfully");
+ proxmox_async::runtime::block_in_place(|| env.remove_backup())?;
+ return Ok(());
+ }
+
+ let verify = |env: BackupEnvironment| {
+ if let Err(err) = env.verify_after_complete(snap_guard) {
+ env.log(format!(
+ "backup finished, but starting the requested verify task failed: {}",
+ err
+ ));
}
+ };
- let verify = |env: BackupEnvironment| {
- if let Err(err) = env.verify_after_complete(snap_guard) {
- env.log(format!(
- "backup finished, but starting the requested verify task failed: {}",
- err
- ));
- }
- };
-
- match (res, env.ensure_finished()) {
- (Ok(_), Ok(())) => {
- env.log("backup finished successfully");
- verify(env);
- Ok(())
- }
- (Err(err), Ok(())) => {
- // ignore errors after finish
- env.log(format!("backup had errors but finished: {}", err));
- verify(env);
- Ok(())
- }
- (Ok(_), Err(err)) => {
- env.log(format!("backup ended and finish failed: {}", err));
- env.log("removing unfinished backup");
- proxmox_async::runtime::block_in_place(|| env.remove_backup())?;
- Err(err)
- }
- (Err(err), Err(_)) => {
- env.log(format!("backup failed: {}", err));
- env.log("removing failed backup");
- proxmox_async::runtime::block_in_place(|| env.remove_backup())?;
- Err(err)
- }
+ match (res, env.ensure_finished()) {
+ (Ok(_), Ok(())) => {
+ env.log("backup finished successfully");
+ verify(env);
+ Ok(())
+ }
+ (Err(err), Ok(())) => {
+ // ignore errors after finish
+ env.log(format!("backup had errors but finished: {}", err));
+ verify(env);
+ Ok(())
+ }
+ (Ok(_), Err(err)) => {
+ env.log(format!("backup ended and finish failed: {}", err));
+ env.log("removing unfinished backup");
+ proxmox_async::runtime::block_in_place(|| env.remove_backup())?;
+ Err(err)
+ }
+ (Err(err), Err(_)) => {
+ env.log(format!("backup failed: {}", err));
+ env.log("removing failed backup");
+ proxmox_async::runtime::block_in_place(|| env.remove_backup())?;
+ Err(err)
}
}
},
--
2.39.5
More information about the pbs-devel
mailing list