[pbs-devel] [PATCH proxmox-backup 1/1] fix #6665: never rename chunks on s3 client fetch errors

[Christian Ebner]

The chunk verification for the s3 backend always fetches the chunk
directly from the backend and verifies it based on the received
response.

Currently, when fetching the chunk contents failed in case of an
unrelated s3 client error, e.g. due to transient networking issues,
both the locally cached and the chunk in the object store, were
incorrectly marked as bad and renamed.

Instead, treat chunk fetching issues as error but do not treat the
chunk itself as bad.

Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=6665
Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
 src/backup/verify.rs | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/src/backup/verify.rs b/src/backup/verify.rs
index b1452f267..069c3bdf9 100644
--- a/src/backup/verify.rs
+++ b/src/backup/verify.rs
@@ -284,13 +284,25 @@ impl VerifyWorker {
                 let object_key = pbs_datastore::s3::object_key_from_digest(&info.digest)?;
                 match proxmox_async::runtime::block_on(s3_client.get_object(object_key)) {
                     Ok(Some(response)) => {
-                        let bytes = proxmox_async::runtime::block_on(response.content.collect())?
-                            .to_bytes();
-                        let chunk = DataBlob::from_raw(bytes.to_vec())?;
-                        let size = info.size();
-                        *read_bytes += chunk.raw_size();
-                        decoder_pool.send((chunk, info.digest, size))?;
-                        *decoded_bytes += size;
+                        let chunk_result = proxmox_lang::try_block!({
+                            let bytes =
+                                proxmox_async::runtime::block_on(response.content.collect())?
+                                    .to_bytes();
+                            DataBlob::from_raw(bytes.to_vec())
+                        });
+
+                        match chunk_result {
+                            Ok(chunk) => {
+                                let size = info.size();
+                                *read_bytes += chunk.raw_size();
+                                decoder_pool.send((chunk, info.digest, size))?;
+                                *decoded_bytes += size;
+                            }
+                            Err(err) => {
+                                errors.fetch_add(1, Ordering::SeqCst);
+                                error!("can't verify chunk, load failed - {err}");
+                            }
+                        }
                     }
                     Ok(None) => self.add_corrupt_chunk(
                         info.digest,
@@ -300,11 +312,10 @@ impl VerifyWorker {
                             hex::encode(info.digest)
                         ),
                     ),
-                    Err(err) => self.add_corrupt_chunk(
-                        info.digest,
-                        errors,
-                        &format!("can't verify chunk, load failed - {err}"),
-                    ),
+                    Err(err) => {
+                        errors.fetch_add(1, Ordering::SeqCst);
+                        error!("can't verify chunk, load failed - {err}");
+                    }
                 }
             }
         }
-- 
2.47.2