[pbs-devel] [PATCH proxmox-backup 4/4] backup: check all referenced chunks actually exist

[Stefan Reiter]

A client can omit uploading chunks in the "known_chunks" list, those
then also won't be written on the server side.  Check all those chunks
mentioned in the index but not uploaded for existance and report an
error if they don't exist instead of marking a potentially broken backup
as "successful".

Signed-off-by: Stefan Reiter <s.reiter at proxmox.com>
---
 src/api2/backup/environment.rs | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 973563d3..df22b1d6 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -1,6 +1,6 @@
 use anyhow::{bail, format_err, Error};
 use std::sync::{Arc, Mutex};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 
 use ::serde::{Serialize};
 use serde_json::{json, Value};
@@ -73,6 +73,7 @@ struct SharedBackupState {
     dynamic_writers: HashMap<usize, DynamicWriterState>,
     fixed_writers: HashMap<usize, FixedWriterState>,
     known_chunks: HashMap<[u8;32], u32>,
+    touched_chunks: HashSet<[u8;32]>,
     backup_size: u64, // sums up size of all files
     backup_stat: UploadStatistic,
 }
@@ -126,6 +127,7 @@ impl BackupEnvironment {
             dynamic_writers: HashMap::new(),
             fixed_writers: HashMap::new(),
             known_chunks: HashMap::new(),
+            touched_chunks: HashSet::new(),
             backup_size: 0,
             backup_stat: UploadStatistic::new(),
         };
@@ -196,6 +198,7 @@ impl BackupEnvironment {
 
         // register chunk
         state.known_chunks.insert(digest, size);
+        state.touched_chunks.insert(digest);
 
         Ok(())
     }
@@ -229,6 +232,7 @@ impl BackupEnvironment {
 
         // register chunk
         state.known_chunks.insert(digest, size);
+        state.touched_chunks.insert(digest);
 
         Ok(())
     }
@@ -490,6 +494,21 @@ impl BackupEnvironment {
             }
         }
 
+        // make sure all chunks that were referenced actually exist
+        for (digest, _) in state.known_chunks.iter() {
+            // if they were uploaded just now they have already been touched
+            if state.touched_chunks.contains(digest) {
+                continue;
+            }
+
+            if !self.datastore.chunk_path(digest).0.exists() {
+                bail!(
+                    "chunk '{}' was attempted to be reused but doesn't exist",
+                    digest_to_hex(digest)
+                );
+            }
+        }
+
         // marks the backup as successful
         state.finished = true;
 
-- 
2.20.1