[pbs-devel] [PATCH proxmox-backup v9 15/46] datastore: local chunk reader: read chunks based on backend

[Lukas Wagner]

On Mon Jul 21, 2025 at 3:12 PM CEST, Hannes Laimer wrote:
>>  #[derive(Clone)]
>>  pub struct LocalChunkReader {
>>      store: Arc<DataStore>,
>> +    backend: DatastoreBackend,
>>      crypt_config: Option<Arc<CryptConfig>>,
>>      crypt_mode: CryptMode,
>>  }
>> @@ -24,8 +28,11 @@ impl LocalChunkReader {
>>          crypt_config: Option<Arc<CryptConfig>>,
>>          crypt_mode: CryptMode,
>>      ) -> Self {
>> +        // TODO: Error handling!
>> +        let backend = store.backend().unwrap();
>
> Was this missed, or was this intentionally left in?
> I feel like we don't want to panic here :P
> (correct me if I'm wrong, but I think we would whenever anything goes
> wrong when connecting to s3?)
>

Good catch! I remember spotting this during my review, but somehow
forgot to note it down in my email, my bad.

>>          Self {
>>              store,
>> +            backend,
>>              crypt_config,
>>              crypt_mode,
>>          }
>> @@ -47,10 +54,26 @@ impl LocalChunkReader {
>>      }
>>  }
>>  
>> +async fn fetch(s3_client: Arc<S3Client>, digest: &[u8; 32]) -> Result<DataBlob, Error> {
>> +    let object_key = crate::s3::object_key_from_digest(digest)?;
>> +    if let Some(response) = s3_client.get_object(object_key).await? {
>> +        let bytes = response.content.collect().await?.to_bytes();
>> +        DataBlob::from_raw(bytes.to_vec())
>> +    } else {
>> +        bail!("no object with digest {}", hex::encode(digest));
>> +    }
>> +}
>> +
>>  impl ReadChunk for LocalChunkReader {
>>      fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
>> -        let chunk = self.store.load_chunk(digest)?;
>> +        let chunk = match &self.backend {
>> +            DatastoreBackend::Filesystem => self.store.load_chunk(digest)?,
>> +            DatastoreBackend::S3(s3_client) => {
>> +                proxmox_async::runtime::block_on(fetch(Arc::clone(s3_client), digest))?
>> +            }
>> +        };
>>          self.ensure_crypt_mode(chunk.crypt_mode()?)?;
>> +
>>          Ok(chunk)
>>      }
>>  
>> @@ -69,11 +92,14 @@ impl AsyncReadChunk for LocalChunkReader {
>>          digest: &'a [u8; 32],
>>      ) -> Pin<Box<dyn Future<Output = Result<DataBlob, Error>> + Send + 'a>> {
>>          Box::pin(async move {
>> -            let (path, _) = self.store.chunk_path(digest);
>> -
>> -            let raw_data = tokio::fs::read(&path).await?;
>> -
>> -            let chunk = DataBlob::load_from_reader(&mut &raw_data[..])?;
>> +            let chunk = match &self.backend {
>> +                DatastoreBackend::Filesystem => {
>> +                    let (path, _) = self.store.chunk_path(digest);
>> +                    let raw_data = tokio::fs::read(&path).await?;
>> +                    DataBlob::load_from_reader(&mut &raw_data[..])?
>> +                }
>> +                DatastoreBackend::S3(s3_client) => fetch(Arc::clone(s3_client), digest).await?,
>> +            };
>>              self.ensure_crypt_mode(chunk.crypt_mode()?)?;
>>  
>>              Ok(chunk)
>
>
>
> _______________________________________________
> pbs-devel mailing list
> pbs-devel at lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel