[pbs-devel] applied: [PATCH proxmox-backup v2 2/4] api2/config/datastore: change create datastore api call to a worker

[Dietmar Maurer]

applied with additional cleanups:

- rename create_datastore_impl to do_create_datastore

- factor out the datastore config lock function

On 6/2/21 1:27 PM, Dominik Csapak wrote:
> so that longer running creates (e.g. a slow storage), does not
> run in a timeout and we can follow its creation
>
> Signed-off-by: Dominik Csapak <d.csapak at proxmox.com>
> ---
>   src/api2/config/datastore.rs     | 51 ++++++++++++++++++++++----------
>   src/api2/node/disks/directory.rs | 15 ++++++++--
>   src/api2/node/disks/zfs.rs       | 14 ++++++++-
>   www/window/DataStoreEdit.js      |  1 +
>   4 files changed, 62 insertions(+), 19 deletions(-)
>
> diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
> index 7299c91d..4fa76b40 100644
> --- a/src/api2/config/datastore.rs
> +++ b/src/api2/config/datastore.rs
> @@ -5,6 +5,7 @@ use serde_json::Value;
>   use ::serde::{Deserialize, Serialize};
>   
>   use proxmox::api::{api, Router, RpcEnvironment, Permission};
> +use proxmox::api::section_config::SectionConfigData;
>   use proxmox::api::schema::parse_property_string;
>   use proxmox::tools::fs::open_file_locked;
>   
> @@ -13,7 +14,7 @@ use crate::backup::*;
>   use crate::config::cached_user_info::CachedUserInfo;
>   use crate::config::datastore::{self, DataStoreConfig, DIR_NAME_SCHEMA};
>   use crate::config::acl::{PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY};
> -use crate::server::jobstate;
> +use crate::server::{jobstate, WorkerTask};
>   
>   #[api(
>       input: {
> @@ -50,6 +51,25 @@ pub fn list_datastores(
>       Ok(list.into_iter().filter(filter_by_privs).collect())
>   }
>   
> +pub(crate) fn create_datastore_impl(
> +    _lock: std::fs::File,
> +    mut config: SectionConfigData,
> +    datastore: DataStoreConfig,
> +) -> Result<(), Error> {
> +    let path: PathBuf = datastore.path.clone().into();
> +
> +    let backup_user = crate::backup::backup_user()?;
> +    let _store = ChunkStore::create(&datastore.name, path, backup_user.uid, backup_user.gid)?;
> +
> +    config.set_data(&datastore.name, "datastore", &datastore)?;
> +
> +    datastore::save_config(&config)?;
> +
> +    jobstate::create_state_file("prune", &datastore.name)?;
> +    jobstate::create_state_file("garbage_collection", &datastore.name)?;
> +
> +    Ok(())
> +}
>   
>   // fixme: impl. const fn get_object_schema(datastore::DataStoreConfig::API_SCHEMA),
>   // but this need support for match inside const fn
> @@ -116,31 +136,30 @@ pub fn list_datastores(
>       },
>   )]
>   /// Create new datastore config.
> -pub fn create_datastore(param: Value) -> Result<(), Error> {
> +pub fn create_datastore(
> +    param: Value,
> +    rpcenv: &mut dyn RpcEnvironment,
> +) -> Result<String, Error> {
>   
> -    let _lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
> +    let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
>   
>       let datastore: datastore::DataStoreConfig = serde_json::from_value(param)?;
>   
> -    let (mut config, _digest) = datastore::config()?;
> +    let (config, _digest) = datastore::config()?;
>   
>       if config.sections.get(&datastore.name).is_some() {
>           bail!("datastore '{}' already exists.", datastore.name);
>       }
>   
> -    let path: PathBuf = datastore.path.clone().into();
> -
> -    let backup_user = crate::backup::backup_user()?;
> -    let _store = ChunkStore::create(&datastore.name, path, backup_user.uid, backup_user.gid)?;
> -
> -    config.set_data(&datastore.name, "datastore", &datastore)?;
> -
> -    datastore::save_config(&config)?;
> -
> -    jobstate::create_state_file("prune", &datastore.name)?;
> -    jobstate::create_state_file("garbage_collection", &datastore.name)?;
> +    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>   
> -    Ok(())
> +    WorkerTask::new_thread(
> +        "create-datastore",
> +        Some(datastore.name.to_string()),
> +        auth_id,
> +        false,
> +        move |_worker| create_datastore_impl(lock, config, datastore),
> +    )
>   }
>   
>   #[api(
> diff --git a/src/api2/node/disks/directory.rs b/src/api2/node/disks/directory.rs
> index c968b333..006e9522 100644
> --- a/src/api2/node/disks/directory.rs
> +++ b/src/api2/node/disks/directory.rs
> @@ -5,6 +5,7 @@ use ::serde::{Deserialize, Serialize};
>   use proxmox::api::{api, Permission, RpcEnvironment, RpcEnvironmentType};
>   use proxmox::api::section_config::SectionConfigData;
>   use proxmox::api::router::Router;
> +use proxmox::tools::fs::open_file_locked;
>   
>   use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
>   use crate::tools::disks::{
> @@ -16,7 +17,7 @@ use crate::tools::systemd::{self, types::*};
>   use crate::server::WorkerTask;
>   
>   use crate::api2::types::*;
> -use crate::config::datastore::DataStoreConfig;
> +use crate::config::datastore::{self, DataStoreConfig};
>   
>   #[api(
>       properties: {
> @@ -179,7 +180,17 @@ pub fn create_datastore_disk(
>               systemd::start_unit(&mount_unit_name)?;
>   
>               if add_datastore {
> -                crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
> +                let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
> +                let datastore: DataStoreConfig =
> +                    serde_json::from_value(json!({ "name": name, "path": mount_point }))?;
> +
> +                let (config, _digest) = datastore::config()?;
> +
> +                if config.sections.get(&datastore.name).is_some() {
> +                    bail!("datastore '{}' already exists.", datastore.name);
> +                }
> +
> +                crate::api2::config::datastore::create_datastore_impl(lock, config, datastore)?;
>               }
>   
>               Ok(())
> diff --git a/src/api2/node/disks/zfs.rs b/src/api2/node/disks/zfs.rs
> index 0f93f110..ee8037e2 100644
> --- a/src/api2/node/disks/zfs.rs
> +++ b/src/api2/node/disks/zfs.rs
> @@ -14,12 +14,14 @@ use proxmox::api::{
>       },
>   };
>   use proxmox::api::router::Router;
> +use proxmox::tools::fs::open_file_locked;
>   
>   use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
>   use crate::tools::disks::{
>       zpool_list, zpool_status, parse_zpool_status_config_tree, vdev_list_to_tree,
>       DiskUsageType,
>   };
> +use crate::config::datastore::{self, DataStoreConfig};
>   
>   use crate::server::WorkerTask;
>   
> @@ -372,7 +374,17 @@ pub fn create_zpool(
>               }
>   
>               if add_datastore {
> -                crate::api2::config::datastore::create_datastore(json!({ "name": name, "path": mount_point }))?
> +                let lock = open_file_locked(datastore::DATASTORE_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?;
> +                let datastore: DataStoreConfig =
> +                    serde_json::from_value(json!({ "name": name, "path": mount_point }))?;
> +
> +                let (config, _digest) = datastore::config()?;
> +
> +                if config.sections.get(&datastore.name).is_some() {
> +                    bail!("datastore '{}' already exists.", datastore.name);
> +                }
> +
> +                crate::api2::config::datastore::create_datastore_impl(lock, config, datastore)?;
>               }
>   
>               Ok(())
> diff --git a/www/window/DataStoreEdit.js b/www/window/DataStoreEdit.js
> index c2b2809f..2f67db76 100644
> --- a/www/window/DataStoreEdit.js
> +++ b/www/window/DataStoreEdit.js
> @@ -75,6 +75,7 @@ Ext.define('PBS.DataStoreEdit', {
>       isAdd: true,
>   
>       bodyPadding: 0,
> +    showProgress: true,
>   
>       cbindData: function(initialConfig) {
>   	var me = this;