[pbs-devel] [PATCH v4 proxmox-backup 03/10] api2: add verification admin endpoint and do_verification_job function
Hannes Laimer
h.laimer at proxmox.com
Tue Oct 20 11:10:05 CEST 2020
Signed-off-by: Hannes Laimer <h.laimer at proxmox.com>
---
src/api2/admin.rs | 4 +-
src/api2/admin/verify.rs | 107 +++++++++++++++++++++++++++++++++++++++
src/backup/verify.rs | 96 +++++++++++++++++++++++++++++++++++
3 files changed, 206 insertions(+), 1 deletion(-)
create mode 100644 src/api2/admin/verify.rs
diff --git a/src/api2/admin.rs b/src/api2/admin.rs
index b927ce1e..79ce29f3 100644
--- a/src/api2/admin.rs
+++ b/src/api2/admin.rs
@@ -3,10 +3,12 @@ use proxmox::list_subdirs_api_method;
pub mod datastore;
pub mod sync;
+pub mod verify;
const SUBDIRS: SubdirMap = &[
("datastore", &datastore::ROUTER),
- ("sync", &sync::ROUTER)
+ ("sync", &sync::ROUTER),
+ ("verify", &verify::ROUTER)
];
pub const ROUTER: Router = Router::new()
diff --git a/src/api2/admin/verify.rs b/src/api2/admin/verify.rs
new file mode 100644
index 00000000..f61373a0
--- /dev/null
+++ b/src/api2/admin/verify.rs
@@ -0,0 +1,107 @@
+use anyhow::{format_err, Error};
+
+use proxmox::api::router::SubdirMap;
+use proxmox::{list_subdirs_api_method, sortable};
+use proxmox::api::{api, ApiMethod, Router, RpcEnvironment};
+
+use crate::api2::types::*;
+use crate::backup::do_verification_job;
+use crate::config::jobstate::{Job, JobState};
+use crate::config::verify;
+use crate::config::verify::{VerificationJobConfig, VerificationJobStatus};
+use serde_json::Value;
+use crate::tools::systemd::time::{parse_calendar_event, compute_next_event};
+use crate::server::UPID;
+
+#[api(
+ input: {
+ properties: {},
+ },
+ returns: {
+ description: "List configured jobs and their status.",
+ type: Array,
+ items: { type: verify::VerificationJobStatus },
+ },
+)]
+/// List all verification jobs
+pub fn list_verification_jobs(
+ _param: Value,
+ mut rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<VerificationJobStatus>, Error> {
+
+ let (config, digest) = verify::config()?;
+
+ let mut list: Vec<VerificationJobStatus> = config.convert_to_typed_array("verification")?;
+
+ for job in &mut list {
+ let last_state = JobState::load("verificationjob", &job.id)
+ .map_err(|err| format_err!("could not open statefile for {}: {}", &job.id, err))?;
+
+ let (upid, endtime, state, starttime) = match last_state {
+ JobState::Created { time } => (None, None, None, time),
+ JobState::Started { upid } => {
+ let parsed_upid: UPID = upid.parse()?;
+ (Some(upid), None, None, parsed_upid.starttime)
+ },
+ JobState::Finished { upid, state } => {
+ let parsed_upid: UPID = upid.parse()?;
+ (Some(upid), Some(state.endtime()), Some(state.to_string()), parsed_upid.starttime)
+ },
+ };
+
+ job.last_run_upid = upid;
+ job.last_run_state = state;
+ job.last_run_endtime = endtime;
+
+ let last = job.last_run_endtime.unwrap_or_else(|| starttime);
+
+ job.next_run = (|| -> Option<i64> {
+ let schedule = job.schedule.as_ref()?;
+ let event = parse_calendar_event(&schedule).ok()?;
+ // ignore errors
+ compute_next_event(&event, last, false).unwrap_or_else(|_| None)
+ })();
+ }
+
+ rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into();
+
+ Ok(list)
+}
+
+#[api(
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ }
+ }
+ }
+)]
+/// Runs a verification job manually.
+fn run_verification_job(
+ id: String,
+ _info: &ApiMethod,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+ let (config, _digest) = verify::config()?;
+ let verification_job: VerificationJobConfig = config.lookup("verification", &id)?;
+
+ let userid: Userid = rpcenv.get_user().unwrap().parse()?;
+
+ let job = Job::new("verificationjob", &id)?;
+
+ let upid_str = do_verification_job(job, verification_job, &userid, None)?;
+
+ Ok(upid_str)
+}
+
+#[sortable]
+const VERIFICATION_INFO_SUBDIRS: SubdirMap = &[("run", &Router::new().post(&API_METHOD_RUN_VERIFICATION_JOB))];
+
+const VERIFICATION_INFO_ROUTER: Router = Router::new()
+ .get(&list_subdirs_api_method!(VERIFICATION_INFO_SUBDIRS))
+ .subdirs(VERIFICATION_INFO_SUBDIRS);
+
+pub const ROUTER: Router = Router::new()
+ .get(&API_METHOD_LIST_VERIFICATION_JOBS)
+ .match_all("id", &VERIFICATION_INFO_ROUTER);
diff --git a/src/backup/verify.rs b/src/backup/verify.rs
index ea3fa760..1d14870a 100644
--- a/src/backup/verify.rs
+++ b/src/backup/verify.rs
@@ -6,7 +6,10 @@ use std::time::Instant;
use anyhow::{bail, format_err, Error};
use crate::{
+ server::WorkerTask,
api2::types::*,
+ config::jobstate::Job,
+ config::verify::VerificationJobConfig,
backup::{
DataStore,
DataBlob,
@@ -504,3 +507,96 @@ pub fn verify_all_backups(
Ok(errors)
}
+
+/// Runs a verification job.
+pub fn do_verification_job(
+ mut job: Job,
+ verification_job: VerificationJobConfig,
+ userid: &Userid,
+ schedule: Option<String>,
+) -> Result<String, Error> {
+ let datastore = DataStore::lookup_datastore(&verification_job.store)?;
+
+ let mut backups_to_verify = BackupInfo::list_backups(&datastore.base_path())?;
+ if verification_job.ignore_verified.unwrap_or(true) {
+ backups_to_verify.retain(|backup_info| {
+ let manifest = match datastore.load_manifest(&backup_info.backup_dir) {
+ Ok((manifest, _)) => manifest,
+ Err(_) => return false,
+ };
+
+ let raw_verify_state = manifest.unprotected["verify_state"].clone();
+ let last_state = match serde_json::from_value::<SnapshotVerifyState>(raw_verify_state) {
+ Ok(last_state) => last_state,
+ Err(_) => return true,
+ };
+
+ let now = proxmox::tools::time::epoch_i64();
+ let days_since_last_verify = (now - last_state.upid.starttime) / 86400;
+ verification_job.outdated_after.is_some()
+ && days_since_last_verify > verification_job.outdated_after.unwrap()
+ })
+ }
+
+ let job_id = job.jobname().to_string();
+ let worker_type = job.jobtype().to_string();
+ let upid_str = WorkerTask::new_thread(
+ &worker_type,
+ Some(job.jobname().to_string()),
+ userid.clone(),
+ false,
+ move |worker| {
+ job.start(&worker.upid().to_string())?;
+
+ task_log!(worker,"Starting datastore verify job '{}'", job_id);
+ task_log!(worker,"verifying {} backups", backups_to_verify.len());
+ if let Some(event_str) = schedule {
+ task_log!(worker,"task triggered by schedule '{}'", event_str);
+ }
+
+ let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024 * 16)));
+ let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64)));
+ let result = proxmox::try_block!({
+ let mut failed_dirs: Vec<String> = Vec::new();
+
+ for backup_info in backups_to_verify {
+ let verification_result = verify_backup_dir(
+ datastore.clone(),
+ &backup_info.backup_dir,
+ verified_chunks.clone(),
+ corrupt_chunks.clone(),
+ worker.clone(),
+ worker.upid().clone()
+ );
+
+ if let Ok(false) = verification_result {
+ failed_dirs.push(backup_info.backup_dir.to_string());
+ } // otherwise successful or aborted
+ }
+
+ if !failed_dirs.is_empty() {
+ task_log!(worker,"Failed to verify following snapshots:",);
+ for dir in failed_dirs {
+ task_log!(worker, "\t{}", dir)
+ }
+ bail!("verification failed - please check the log for details");
+ }
+ Ok(())
+ });
+
+ let status = worker.create_state(&result);
+
+ match job.finish(status) {
+ Err(err) => eprintln!(
+ "could not finish job state for {}: {}",
+ job.jobtype().to_string(),
+ err
+ ),
+ Ok(_) => (),
+ }
+
+ result
+ },
+ )?;
+ Ok(upid_str)
+}
--
2.20.1
More information about the pbs-devel
mailing list