[pbs-devel] [RFC proxmox-backup 19/39] api: backup: conditionally upload manifest to S3 object store backend

[Christian Ebner]

Upload the manifest to the S3 object store backend after it has been
finished in the backup api call handler, if s3 is configured as
backend. Keep also the locally cached version for fast and efficient
listing of contents without the need to perform expensive (as in
monetary cost and IO latency) requests.

The datastore's metadata contents will be synced from the S3 backend
during datastore opening.

Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
 src/api2/backup/environment.rs | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 72e369bcf..685b78e89 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -12,7 +12,7 @@ use serde_json::{json, Value};
 use proxmox_router::{RpcEnvironment, RpcEnvironmentType};
 use proxmox_sys::fs::{replace_file, CreateOptions};
 
-use pbs_api_types::Authid;
+use pbs_api_types::{Authid, MANIFEST_BLOB_NAME};
 use pbs_datastore::backup_info::{BackupDir, BackupInfo};
 use pbs_datastore::dynamic_index::DynamicIndexWriter;
 use pbs_datastore::fixed_index::FixedIndexWriter;
@@ -719,6 +719,37 @@ impl BackupEnvironment {
             }
         }
 
+        if let DatastoreBackend::S3(s3_client) = &self.backend {
+            // Upload manifest to S3 object store
+            let mut object_key = self.backup_dir.relative_path();
+            object_key.push(MANIFEST_BLOB_NAME.as_ref());
+            let mut path = self.datastore.base_path();
+            path.push(&object_key);
+            let mut manifest = std::fs::File::open(&path)?;
+            let mut buffer = Vec::new();
+            manifest.read_to_end(&mut buffer)?;
+            let data = Body::from(buffer);
+            let object_key = object_key
+                .as_os_str()
+                .to_str()
+                .ok_or_else(|| format_err!("invalid path"))?;
+            match proxmox_async::runtime::block_on(s3_client.put_object(object_key.into(), data))? {
+                PutObjectResponse::PreconditionFailed => {
+                    self.log(format!(
+                        "Upload of manifest failed, object {object_key} already present."
+                    ));
+                    bail!("upload of manifest failed");
+                }
+                PutObjectResponse::NeedsRetry => {
+                    self.log("Upload of manifest failed, reupload required.");
+                    bail!("concurrent operation, reupload required");
+                }
+                PutObjectResponse::Success(_content) => {
+                    self.log(format!("Uploaded manifest to object store: {object_key}"))
+                }
+            }
+        }
+
         self.datastore.try_ensure_sync_level()?;
 
         // marks the backup as successful
-- 
2.39.5