[pbs-devel] [PATCH proxmox-backup v8 02/45] config: introduce s3 object store client configuration

[Lukas Wagner]

With my minor complaints fixed:

Reviewed-by: Lukas Wagner <l.wagner at proxmox.com>

On  2025-07-15 14:52, Christian Ebner wrote:
> Adds the client configuration for s3 object store as dedicated
> configuration files, with secrets being stored separately from the
> regular configuration and excluded from api responses for security
> reasons.
> 
> Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
> ---
> changes since version 7:
> - no changes
> 
>  pbs-config/Cargo.toml |  1 +
>  pbs-config/src/lib.rs |  1 +
>  pbs-config/src/s3.rs  | 83 +++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 85 insertions(+)
>  create mode 100644 pbs-config/src/s3.rs
> 
> diff --git a/pbs-config/Cargo.toml b/pbs-config/Cargo.toml
> index 284149658..74afb3c64 100644
> --- a/pbs-config/Cargo.toml
> +++ b/pbs-config/Cargo.toml
> @@ -19,6 +19,7 @@ serde_json.workspace = true
>  
>  proxmox-notify.workspace = true
>  proxmox-router = { workspace = true, default-features = false }
> +proxmox-s3-client.workspace = true
>  proxmox-schema.workspace = true
>  proxmox-section-config.workspace = true
>  proxmox-shared-memory.workspace = true
> diff --git a/pbs-config/src/lib.rs b/pbs-config/src/lib.rs
> index 9c4d77c24..d03c079ab 100644
> --- a/pbs-config/src/lib.rs
> +++ b/pbs-config/src/lib.rs
> @@ -10,6 +10,7 @@ pub mod network;
>  pub mod notifications;
>  pub mod prune;
>  pub mod remote;
> +pub mod s3;
>  pub mod sync;
>  pub mod tape_job;
>  pub mod token_shadow;
> diff --git a/pbs-config/src/s3.rs b/pbs-config/src/s3.rs
> new file mode 100644
> index 000000000..ec3998834
> --- /dev/null
> +++ b/pbs-config/src/s3.rs
> @@ -0,0 +1,83 @@
> +use std::collections::HashMap;
> +use std::sync::LazyLock;
> +
> +use anyhow::Error;
> +
> +use proxmox_s3_client::{S3ClientConfig, S3ClientSecretsConfig};
> +use proxmox_schema::*;
> +use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
> +
> +use pbs_api_types::JOB_ID_SCHEMA;
> +
> +use crate::{open_backup_lockfile, replace_backup_config, BackupLockGuard};
> +
> +pub static CONFIG: LazyLock<SectionConfig> = LazyLock::new(init);
> +
> +fn init() -> SectionConfig {
> +    let obj_schema = match S3ClientConfig::API_SCHEMA {
> +        Schema::Object(ref obj_schema) => obj_schema,
> +        _ => unreachable!(),
> +    };
> +    let secrets_obj_schema = match S3ClientSecretsConfig::API_SCHEMA {
> +        Schema::Object(ref obj_schema) => obj_schema,
> +        _ => unreachable!(),
> +    };

You can use API_SCHEMA::unwrap_object_schema here, that's a bit nicer to read :)

> +
> +    let plugin =
> +        SectionConfigPlugin::new("s3client".to_string(), Some(String::from("id")), obj_schema);
> +    let secrets_plugin = SectionConfigPlugin::new(
> +        "s3secrets".to_string(),
> +        Some(String::from("secrets-id")),
> +        secrets_obj_schema,
> +    );
> +    let mut config = SectionConfig::new(&JOB_ID_SCHEMA);
> +    config.register_plugin(plugin);
> +    config.register_plugin(secrets_plugin);
> +
> +    config
> +}
> +
> +pub const S3_CFG_FILENAME: &str = "/etc/proxmox-backup/s3.cfg";
> +pub const S3_SECRETS_CFG_FILENAME: &str = "/etc/proxmox-backup/s3-secrets.cfg";
> +pub const S3_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.s3.lck";

You can use the pbs_buildcfg::configdir macro to build these paths. Also please
add some docstrings to public consts like these.

> +
> +/// Get exclusive lock
> +pub fn lock_config() -> Result<BackupLockGuard, Error> {
> +    open_backup_lockfile(S3_CFG_LOCKFILE, None, true)
> +}
> +
> +pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> {
> +    parse_config(S3_CFG_FILENAME)
> +}
> +
> +pub fn secrets_config() -> Result<(SectionConfigData, [u8; 32]), Error> {
> +    parse_config(S3_SECRETS_CFG_FILENAME)
> +}
> +
> +pub fn save_config(config: &SectionConfigData, secrets: &SectionConfigData) -> Result<(), Error> {
> +    let raw = CONFIG.write(S3_CFG_FILENAME, config)?;
> +    replace_backup_config(S3_CFG_FILENAME, raw.as_bytes())?;
> +
> +    let secrets_raw = CONFIG.write(S3_SECRETS_CFG_FILENAME, secrets)?;
> +    // Secrets are stored with `backup` permissions to allow reading from
> +    // not protected api endpoints as well.
> +    replace_backup_config(S3_SECRETS_CFG_FILENAME, secrets_raw.as_bytes())?;
> +
> +    Ok(())
> +}

^ These public functions lack docstrings

> +
> +// shell completion helper
> +pub fn complete_s3_client_id(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
> +    match config() {
> +        Ok((data, _digest)) => data.sections.keys().map(|id| id.to_string()).collect(),
> +        Err(_) => Vec::new(),
> +    }
> +}
> +
> +fn parse_config(path: &str) -> Result<(SectionConfigData, [u8; 32]), Error> {
> +    let content = proxmox_sys::fs::file_read_optional_string(path)?;
> +    let content = content.unwrap_or_default();
> +    let digest = openssl::sha::sha256(content.as_bytes());
> +    let data = CONFIG.parse(path, &content)?;
> +    Ok((data, digest))
> +}

-- 
- Lukas