[pbs-devel] [PATCH proxmox-backup v8 02/45] config: introduce s3 object store client configuration

[Christian Ebner]

On 7/18/25 9:22 AM, Lukas Wagner wrote:
> With my minor complaints fixed:
> 
> Reviewed-by: Lukas Wagner <l.wagner at proxmox.com>
> 
> On  2025-07-15 14:52, Christian Ebner wrote:
>> Adds the client configuration for s3 object store as dedicated
>> configuration files, with secrets being stored separately from the
>> regular configuration and excluded from api responses for security
>> reasons.
>>
>> Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
>> ---
>> changes since version 7:
>> - no changes
>>
>>   pbs-config/Cargo.toml |  1 +
>>   pbs-config/src/lib.rs |  1 +
>>   pbs-config/src/s3.rs  | 83 +++++++++++++++++++++++++++++++++++++++++++
>>   3 files changed, 85 insertions(+)
>>   create mode 100644 pbs-config/src/s3.rs
>>
>> diff --git a/pbs-config/Cargo.toml b/pbs-config/Cargo.toml
>> index 284149658..74afb3c64 100644
>> --- a/pbs-config/Cargo.toml
>> +++ b/pbs-config/Cargo.toml
>> @@ -19,6 +19,7 @@ serde_json.workspace = true
>>   
>>   proxmox-notify.workspace = true
>>   proxmox-router = { workspace = true, default-features = false }
>> +proxmox-s3-client.workspace = true
>>   proxmox-schema.workspace = true
>>   proxmox-section-config.workspace = true
>>   proxmox-shared-memory.workspace = true
>> diff --git a/pbs-config/src/lib.rs b/pbs-config/src/lib.rs
>> index 9c4d77c24..d03c079ab 100644
>> --- a/pbs-config/src/lib.rs
>> +++ b/pbs-config/src/lib.rs
>> @@ -10,6 +10,7 @@ pub mod network;
>>   pub mod notifications;
>>   pub mod prune;
>>   pub mod remote;
>> +pub mod s3;
>>   pub mod sync;
>>   pub mod tape_job;
>>   pub mod token_shadow;
>> diff --git a/pbs-config/src/s3.rs b/pbs-config/src/s3.rs
>> new file mode 100644
>> index 000000000..ec3998834
>> --- /dev/null
>> +++ b/pbs-config/src/s3.rs
>> @@ -0,0 +1,83 @@
>> +use std::collections::HashMap;
>> +use std::sync::LazyLock;
>> +
>> +use anyhow::Error;
>> +
>> +use proxmox_s3_client::{S3ClientConfig, S3ClientSecretsConfig};
>> +use proxmox_schema::*;
>> +use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
>> +
>> +use pbs_api_types::JOB_ID_SCHEMA;
>> +
>> +use crate::{open_backup_lockfile, replace_backup_config, BackupLockGuard};
>> +
>> +pub static CONFIG: LazyLock<SectionConfig> = LazyLock::new(init);
>> +
>> +fn init() -> SectionConfig {
>> +    let obj_schema = match S3ClientConfig::API_SCHEMA {
>> +        Schema::Object(ref obj_schema) => obj_schema,
>> +        _ => unreachable!(),
>> +    };
>> +    let secrets_obj_schema = match S3ClientSecretsConfig::API_SCHEMA {
>> +        Schema::Object(ref obj_schema) => obj_schema,
>> +        _ => unreachable!(),
>> +    };
> 
> You can use API_SCHEMA::unwrap_object_schema here, that's a bit nicer to read :)

agreed, incorporated this for the next iteration of the patches

>> +
>> +    let plugin =
>> +        SectionConfigPlugin::new("s3client".to_string(), Some(String::from("id")), obj_schema);
>> +    let secrets_plugin = SectionConfigPlugin::new(
>> +        "s3secrets".to_string(),
>> +        Some(String::from("secrets-id")),
>> +        secrets_obj_schema,
>> +    );
>> +    let mut config = SectionConfig::new(&JOB_ID_SCHEMA);
>> +    config.register_plugin(plugin);
>> +    config.register_plugin(secrets_plugin);
>> +
>> +    config
>> +}
>> +
>> +pub const S3_CFG_FILENAME: &str = "/etc/proxmox-backup/s3.cfg";
>> +pub const S3_SECRETS_CFG_FILENAME: &str = "/etc/proxmox-backup/s3-secrets.cfg";
>> +pub const S3_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.s3.lck";
> 
> You can use the pbs_buildcfg::configdir macro to build these paths. Also please
> add some docstrings to public consts like these.

same, added the helper so we always use the configured path from 
buildcfg as base path for these constans.

> 
>> +
>> +/// Get exclusive lock
>> +pub fn lock_config() -> Result<BackupLockGuard, Error> {
>> +    open_backup_lockfile(S3_CFG_LOCKFILE, None, true)
>> +}
>> +
>> +pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> {
>> +    parse_config(S3_CFG_FILENAME)
>> +}
>> +
>> +pub fn secrets_config() -> Result<(SectionConfigData, [u8; 32]), Error> {
>> +    parse_config(S3_SECRETS_CFG_FILENAME)
>> +}
>> +
>> +pub fn save_config(config: &SectionConfigData, secrets: &SectionConfigData) -> Result<(), Error> {
>> +    let raw = CONFIG.write(S3_CFG_FILENAME, config)?;
>> +    replace_backup_config(S3_CFG_FILENAME, raw.as_bytes())?;
>> +
>> +    let secrets_raw = CONFIG.write(S3_SECRETS_CFG_FILENAME, secrets)?;
>> +    // Secrets are stored with `backup` permissions to allow reading from
>> +    // not protected api endpoints as well.
>> +    replace_backup_config(S3_SECRETS_CFG_FILENAME, secrets_raw.as_bytes())?;
>> +
>> +    Ok(())
>> +}
> 
> ^ These public functions lack docstrings

added these as well ...

> 
>> +
>> +// shell completion helper

... and expanded a bit on this one since already at it.

>> +pub fn complete_s3_client_id(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
>> +    match config() {
>> +        Ok((data, _digest)) => data.sections.keys().map(|id| id.to_string()).collect(),
>> +        Err(_) => Vec::new(),
>> +    }
>> +}
>> +
>> +fn parse_config(path: &str) -> Result<(SectionConfigData, [u8; 32]), Error> {
>> +    let content = proxmox_sys::fs::file_read_optional_string(path)?;
>> +    let content = content.unwrap_or_default();
>> +    let digest = openssl::sha::sha256(content.as_bytes());
>> +    let data = CONFIG.parse(path, &content)?;
>> +    Ok((data, digest))
>> +}
>