[pbs-devel] [PATCH proxmox-backup v7 32/38] api: reader: use local datastore cache on s3 backend chunk fetching

[Christian Ebner]

Take advantage of the local datastore filesystem cache for datastores
backed by an s3 object store in order to reduce number of requests
and latency, and increase throughput.

Also, reducing the number of requests is cost beneficial for S3 object
stores charging for fetching of objects.

Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
changes since version 6:
- no changes

 src/api2/reader/mod.rs | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/api2/reader/mod.rs b/src/api2/reader/mod.rs
index 997d9ca77..846493c61 100644
--- a/src/api2/reader/mod.rs
+++ b/src/api2/reader/mod.rs
@@ -327,7 +327,28 @@ fn download_chunk(
 
         let body = match &env.backend {
             DatastoreBackend::Filesystem => load_from_filesystem(env, &digest)?,
-            DatastoreBackend::S3(s3_client) => fetch_from_object_store(s3_client, &digest).await?,
+            DatastoreBackend::S3(s3_client) => {
+                match env.datastore.cache() {
+                    None => fetch_from_object_store(s3_client, &digest).await?,
+                    Some(cache) => {
+                        let mut cacher = env
+                            .datastore
+                            .cacher()?
+                            .ok_or(format_err!("no cacher for datastore"))?;
+                        // Download from object store, insert to local cache store and read from
+                        // file. Can this be optimized?
+                        let chunk =
+                            cache
+                                .access(&digest, &mut cacher)
+                                .await?
+                                .ok_or(format_err!(
+                                    "unable to access chunk with digest {}",
+                                    hex::encode(digest)
+                                ))?;
+                        Body::from(chunk.raw_data().to_owned())
+                    }
+                }
+            }
         };
 
         // fixme: set other headers ?
-- 
2.47.2