[pbs-devel] [PATCH proxmox-backup v11 10/46] api: backup: conditionally upload indices to s3 object store backend

[Christian Ebner]

If the datastore is backed by an S3 compatible object store, upload
the dynamic or fixed index files to the object store after closing
them. The local index files are kept in the local caching datastore
to allow for fast and efficient content lookups, avoiding expensive
(as in monetary cost and IO latency) requests.

Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
Reviewed-by: Lukas Wagner <l.wagner at proxmox.com>
Reviewed-by: Hannes Laimer <h.laimer at proxmox.com>
---
changes since version 10:
 - no changes

 src/api2/backup/environment.rs | 36 ++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index a5683b845..b67145fa6 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -18,6 +18,7 @@ use pbs_datastore::dynamic_index::DynamicIndexWriter;
 use pbs_datastore::fixed_index::FixedIndexWriter;
 use pbs_datastore::{DataBlob, DataStore, DatastoreBackend};
 use proxmox_rest_server::{formatter::*, WorkerTask};
+use proxmox_s3_client::S3Client;
 
 use crate::backup::VerifyWorker;
 
@@ -479,6 +480,16 @@ impl BackupEnvironment {
             );
         }
 
+        // For S3 backends, upload the index file to the object store after closing
+        if let DatastoreBackend::S3(s3_client) = &self.backend {
+            self.s3_upload_index(s3_client, &data.name)
+                .context("failed to upload dynamic index to s3 backend")?;
+            self.log(format!(
+                "Uploaded dynamic index file to s3 backend: {}",
+                data.name
+            ))
+        }
+
         self.log_upload_stat(
             &data.name,
             &csum,
@@ -553,6 +564,16 @@ impl BackupEnvironment {
             );
         }
 
+        // For S3 backends, upload the index file to the object store after closing
+        if let DatastoreBackend::S3(s3_client) = &self.backend {
+            self.s3_upload_index(s3_client, &data.name)
+                .context("failed to upload fixed index to s3 backend")?;
+            self.log(format!(
+                "Uploaded fixed index file to object store: {}",
+                data.name
+            ))
+        }
+
         self.log_upload_stat(
             &data.name,
             &expected_csum,
@@ -751,6 +772,21 @@ impl BackupEnvironment {
 
         Ok(())
     }
+
+    fn s3_upload_index(&self, s3_client: &S3Client, name: &str) -> Result<(), Error> {
+        let object_key =
+            pbs_datastore::s3::object_key_from_path(&self.backup_dir.relative_path(), name)
+                .context("invalid index file object key")?;
+
+        let mut full_path = self.backup_dir.full_path();
+        full_path.push(name);
+        let data = std::fs::read(&full_path).context("failed to read index contents")?;
+        let contents = hyper::body::Bytes::from(data);
+        proxmox_async::runtime::block_on(
+            s3_client.upload_replace_with_retry(object_key, contents),
+        )?;
+        Ok(())
+    }
 }
 
 impl RpcEnvironment for BackupEnvironment {
-- 
2.47.2