[pbs-devel] [PATCH proxmox-backup 3/4] datastore: s3: set rate limiter options for s3 client

[Christian Ebner]

Set the shared rate limiter for each client instance based on the
endpoint configuration. The same limits are shared for each s3
endpoint. To avoid possibly id clashing with rate limits set via
traffic control, use the base directory `<rundir>/s3/shmem/tbf`
instead of the traffic control's `<rundir>/shmem/tbf`.

Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
 pbs-datastore/src/datastore.rs | 18 +++++++++++++++++-
 src/api2/admin/s3.rs           |  9 +++++++--
 src/api2/config/s3.rs          |  2 +-
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 7cf020fc0..e7cc76dc7 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -14,7 +14,9 @@ use tokio::io::AsyncWriteExt;
 use tracing::{info, warn};
 
 use proxmox_human_byte::HumanByte;
-use proxmox_s3_client::{S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix};
+use proxmox_s3_client::{
+    S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix, S3RateLimiterOptions,
+};
 use proxmox_schema::ApiType;
 
 use proxmox_sys::error::SysError;
@@ -55,6 +57,7 @@ pub const GROUP_NOTES_FILE_NAME: &str = "notes";
 pub const GROUP_OWNER_FILE_NAME: &str = "owner";
 /// Filename for in-use marker stored on S3 object store backend
 pub const S3_DATASTORE_IN_USE_MARKER: &str = ".in-use";
+const S3_CLIENT_RATE_LIMITER_BASE_PATH: &str = pbs_buildcfg::rundir!("/s3/shmem/tbf");
 const NAMESPACE_MARKER_FILENAME: &str = ".namespace";
 
 /// checks if auth_id is owner, or, if owner is a token, if
@@ -254,12 +257,18 @@ impl DataStore {
 
                 let (config, _config_digest) = pbs_config::s3::config()?;
                 let config: S3ClientConf = config.lookup(S3_CFG_TYPE_ID, s3_client_id)?;
+                let rate_limiter_options = S3RateLimiterOptions {
+                    id: s3_client_id.to_string(),
+                    user: pbs_config::backup_user()?,
+                    base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(),
+                };
 
                 let options = S3ClientOptions::from_config(
                     config.config,
                     config.secret_key,
                     Some(bucket),
                     self.name().to_owned(),
+                    Some(rate_limiter_options),
                 );
                 let s3_client = S3Client::new(options)?;
                 DatastoreBackend::S3(Arc::new(s3_client))
@@ -2432,11 +2441,18 @@ impl DataStore {
         let client_config: S3ClientConf = config
             .lookup(S3_CFG_TYPE_ID, s3_client_id)
             .with_context(|| format!("no '{s3_client_id}' in config"))?;
+        let rate_limiter_options = S3RateLimiterOptions {
+            id: s3_client_id.to_string(),
+            user: pbs_config::backup_user()?,
+            base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(),
+        };
+
         let options = S3ClientOptions::from_config(
             client_config.config,
             client_config.secret_key,
             Some(bucket),
             datastore_config.name.to_owned(),
+            Some(rate_limiter_options),
         );
         let s3_client = S3Client::new(options)
             .context("failed to create s3 client")
diff --git a/src/api2/admin/s3.rs b/src/api2/admin/s3.rs
index 73f3779a5..73388281b 100644
--- a/src/api2/admin/s3.rs
+++ b/src/api2/admin/s3.rs
@@ -49,8 +49,13 @@ pub async fn check(
         .context("config lookup failed")?;
 
     let store_prefix = store_prefix.unwrap_or_default();
-    let options =
-        S3ClientOptions::from_config(config.config, config.secret_key, Some(bucket), store_prefix);
+    let options = S3ClientOptions::from_config(
+        config.config,
+        config.secret_key,
+        Some(bucket),
+        store_prefix,
+        None,
+    );
 
     let test_object_key =
         S3ObjectKey::try_from(".s3-client-test").context("failed to generate s3 object key")?;
diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs
index 1e421114c..27b3c4cc2 100644
--- a/src/api2/config/s3.rs
+++ b/src/api2/config/s3.rs
@@ -351,7 +351,7 @@ pub async fn list_buckets(
 
     let empty_prefix = String::new();
     let options =
-        S3ClientOptions::from_config(config.config, config.secret_key, None, empty_prefix);
+        S3ClientOptions::from_config(config.config, config.secret_key, None, empty_prefix, None);
     let client = S3Client::new(options).context("client creation failed")?;
     let list_buckets_response = client
         .list_buckets()
-- 
2.47.2