[pbs-devel] [PATCH proxmox-backup v8 33/45] datastore: local chunk reader: get cached chunk from local cache store

[Christian Ebner]

On 7/18/25 1:36 PM, Lukas Wagner wrote:
> With my nits addresses:
> 
> Reviewed-by: Lukas Wagner <l.wagner at proxmox.com>
> 
> 
> On  2025-07-15 14:53, Christian Ebner wrote:
>> Check if a chunk is contained in the local cache and if so prefer
>> fetching it from the cache instead of pulling it via the S3 api. This
>> improves performance and reduces number of requests to the backend.
>>
>> Basic restore performance tests:
>>
>> Restored a snapshot containing the linux git repository (on-disk size
>> 5.069 GiB, compressed 3.718 GiB) from an AWS S3 backed datastore, with
>> and without cached contents:
>> non cached: 691.95 s
>> all cached:  74.89 s
>>
>> Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
>> ---
>> changes since version 7:
>> - no changes
>>
>>   pbs-datastore/src/local_chunk_reader.rs | 31 +++++++++++++++++++++----
>>   1 file changed, 26 insertions(+), 5 deletions(-)
>>
>> diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs
>> index f5aa217ae..7ad44c4fa 100644
>> --- a/pbs-datastore/src/local_chunk_reader.rs
>> +++ b/pbs-datastore/src/local_chunk_reader.rs
>> @@ -2,7 +2,7 @@ use std::future::Future;
>>   use std::pin::Pin;
>>   use std::sync::Arc;
>>   
>> -use anyhow::{bail, Error};
>> +use anyhow::{bail, format_err, Error};
>>   use http_body_util::BodyExt;
>>   
>>   use pbs_api_types::CryptMode;
>> @@ -68,9 +68,18 @@ impl ReadChunk for LocalChunkReader {
>>       fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
>>           let chunk = match &self.backend {
>>               DatastoreBackend::Filesystem => self.store.load_chunk(digest)?,
>> -            DatastoreBackend::S3(s3_client) => {
>> -                proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))?
> 
> better use Arc::clone here :)

adapted here ...

> 
>> -            }
>> +            DatastoreBackend::S3(s3_client) => match self.store.cache() {
>> +                None => proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))?,
>> +                Some(cache) => {
>> +                    let mut cacher = self
>> +                        .store
>> +                        .cacher()?
>> +                        .ok_or(format_err!("no cacher for datastore"))?;
>> +                    proxmox_async::runtime::block_on(cache.access(digest, &mut cacher))?.ok_or(
>> +                        format_err!("unable to access chunk with digest {}", hex::encode(digest)),
>> +                    )?
>> +                }
>> +            },
>>           };
>>           self.ensure_crypt_mode(chunk.crypt_mode()?)?;
>>   
>> @@ -98,7 +107,19 @@ impl AsyncReadChunk for LocalChunkReader {
>>                       let raw_data = tokio::fs::read(&path).await?;
>>                       DataBlob::load_from_reader(&mut &raw_data[..])?
>>                   }
>> -                DatastoreBackend::S3(s3_client) => fetch(s3_client.clone(), digest).await?,
>> +                DatastoreBackend::S3(s3_client) => match self.store.cache() {
>> +                    None => fetch(s3_client.clone(), digest).await?,
> 
> same here

... and here

> 
>> +                    Some(cache) => {
>> +                        let mut cacher = self
>> +                            .store
>> +                            .cacher()?
>> +                            .ok_or(format_err!("no cacher for datastore"))?;
>> +                        cache.access(digest, &mut cacher).await?.ok_or(format_err!(
>> +                            "unable to access chunk with digest {}",
>> +                            hex::encode(digest)
>> +                        ))?
>> +                    }
>> +                },
>>               };
>>               self.ensure_crypt_mode(chunk.crypt_mode()?)?;
>>   
>