[pbs-devel] [RFC PATCH proxmox-backup v2 2/3] verify: add tuning option for number of threads to use

[Dominik Csapak]

a similar argument as for tape backups, but limited to the hashing part
of verification.

The ParallelHandler we use here only parallelizes the actual
verification, not the reading of chunks.

So in case the CPU is much slower than reading from storage, this could
help increase throughput.

In my benchmarks, the only difference i saw was when the cpu was limited
and I reduced the number of threads. So i guess this knob could help in
situations where the single-threaded read performance of the storage
outperforms the 4 cpu threads in hashing.

Because of that, I left the default at '4' threads like before.

Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
---
 pbs-api-types/src/datastore.rs | 7 +++++++
 pbs-datastore/src/datastore.rs | 5 +++++
 src/backup/verify.rs           | 4 +++-
 www/Utils.js                   | 5 +++++
 www/datastore/OptionView.js    | 8 ++++++++
 5 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs
index 1dae3867f..3fb9ff766 100644
--- a/pbs-api-types/src/datastore.rs
+++ b/pbs-api-types/src/datastore.rs
@@ -214,6 +214,11 @@ pub enum DatastoreFSyncLevel {
             optional: true,
             minimum: 1,
         },
+        "verification-threads": {
+            type: usize,
+            optional: true,
+            minimum: 1,
+        },
     },
 )]
 #[derive(Serialize, Deserialize, Default)]
@@ -228,6 +233,8 @@ pub struct DatastoreTuning {
     #[serde(skip_serializing_if = "Option::is_none")]
     /// Configures how many threads to use to read from the datastore while backing up to tape.
     pub tape_backup_read_threads: Option<usize>,
+    /// Configures how many threads to use for hashing on verify.
+    pub verification_threads: Option<usize>,
 }
 
 pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore tuning options")
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 73a1cfa39..b6552d92c 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -52,12 +52,14 @@ pub fn check_backup_owner(owner: &Authid, auth_id: &Authid) -> Result<(), Error>
 /// Contains the configuration of how many threads to use in various situations
 pub struct ThreadConfiguration {
     pub tape_backup_read_threads: usize,
+    pub verification_threads: usize,
 }
 
 impl Default for ThreadConfiguration {
     fn default() -> Self {
         Self {
             tape_backup_read_threads: 4,
+            verification_threads: 4,
         }
     }
 }
@@ -324,6 +326,9 @@ impl DataStore {
         if let Some(value) = tuning.tape_backup_read_threads {
             thread_config.tape_backup_read_threads = value;
         }
+        if let Some(value) = tuning.verification_threads {
+            thread_config.verification_threads = value;
+        }
 
         Ok(DataStoreImpl {
             chunk_store,
diff --git a/src/backup/verify.rs b/src/backup/verify.rs
index c972e5328..c9fbb2e33 100644
--- a/src/backup/verify.rs
+++ b/src/backup/verify.rs
@@ -123,9 +123,11 @@ fn verify_index_chunks(
     let verified_chunks2 = Arc::clone(&verify_worker.verified_chunks);
     let errors2 = Arc::clone(&errors);
 
+    let thread_count = datastore2.get_thread_configuration().verification_threads;
+
     let decoder_pool = ParallelHandler::new(
         "verify chunk decoder",
-        4,
+        thread_count,
         move |(chunk, digest, size): (DataBlob, [u8; 32], u64)| {
             let chunk_crypt_mode = match chunk.crypt_mode() {
                 Err(err) => {
diff --git a/www/Utils.js b/www/Utils.js
index 4d224cd4a..a94bdf86e 100644
--- a/www/Utils.js
+++ b/www/Utils.js
@@ -795,6 +795,11 @@ Ext.define('PBS.Utils', {
 	tapeBackupRT ??= Proxmox.Utils.defaultText + ` (4)`;
 	options.push(`${gettext('Tape Backup Read Threads')}: ${tapeBackupRT}`);
 
+	let verificationThreads = tuning['verification-threads'];
+	delete tuning['verification-threads'];
+	verificationThreads ??= Proxmox.Utils.defaultText + ` (4)`;
+	options.push(`${gettext('Verification Threads')}: ${verificationThreads}`);
+
 	for (const [k, v] of Object.entries(tuning)) {
 	    options.push(`${k}: ${v}`);
 	}
diff --git a/www/datastore/OptionView.js b/www/datastore/OptionView.js
index cfbb89151..c28294bfd 100644
--- a/www/datastore/OptionView.js
+++ b/www/datastore/OptionView.js
@@ -279,6 +279,14 @@ Ext.define('PBS.Datastore.Options', {
 			    emptyText: '4',
 			    deleteEmpty: true,
 			},
+			{
+			    xtype: 'proxmoxintegerfield',
+			    name: 'verification-threads',
+			    fieldLabel: gettext('Verification Threads'),
+			    min: 1,
+			    emptyText: '4',
+			    deleteEmpty: true,
+			},
 		    ],
 		},
 	    },
-- 
2.39.2