[pve-devel] [pbs-devel] [PATCH proxmox v2 01/12] notify: implement webhook targets
Max Carrara
m.carrara at proxmox.com
Wed Jul 17 17:35:00 CEST 2024
On Fri Jul 12, 2024 at 1:27 PM CEST, Lukas Wagner wrote:
> This target type allows users to perform HTTP requests to arbitrary
> third party (notification) services, for instance
> ntfy.sh/Discord/Slack.
>
> The configuration for these endpoints allows one to freely configure
> the URL, HTTP Method, headers and body. The URL, header values and
> body support handlebars templating to inject notification text,
> metadata and secrets. Secrets are stored in the protected
> configuration file (e.g. /etc/pve/priv/notification.cfg) as key value
> pairs, allowing users to protect sensitive tokens/passwords.
> Secrets are accessible in handlebar templating via the secrets.*
> namespace, e.g. if there is a secret named 'token', a body
> could contain '{{ secrets.token }}' to inject the token into the
> payload.
>
> A couple of handlebars helpers are also provided:
> - url-encoding (useful for templating in URLs)
> - escape (escape any control characters in strings)
> - json (print a property as json)
>
> In the configuration, the body, header values and secret values
> are stored in base64 encoding so that we can store any string we want.
>
> Signed-off-by: Lukas Wagner <l.wagner at proxmox.com>
> ---
> proxmox-notify/Cargo.toml | 9 +-
> proxmox-notify/src/config.rs | 23 ++
> proxmox-notify/src/endpoints/mod.rs | 2 +
> proxmox-notify/src/endpoints/webhook.rs | 509 ++++++++++++++++++++++++
> proxmox-notify/src/lib.rs | 17 +
> 5 files changed, 557 insertions(+), 3 deletions(-)
> create mode 100644 proxmox-notify/src/endpoints/webhook.rs
>
> diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml
> index 7801814d..484aff19 100644
> --- a/proxmox-notify/Cargo.toml
> +++ b/proxmox-notify/Cargo.toml
> @@ -9,13 +9,15 @@ exclude.workspace = true
>
> [dependencies]
> anyhow.workspace = true
> -base64.workspace = true
> +base64 = { workspace = true, optional = true }
> const_format.workspace = true
> handlebars = { workspace = true }
> +http = { workspace = true, optional = true }
> lettre = { workspace = true, optional = true }
> log.workspace = true
> mail-parser = { workspace = true, optional = true }
> openssl.workspace = true
> +percent-encoding = { workspace = true, optional = true }
> regex.workspace = true
> serde = { workspace = true, features = ["derive"] }
> serde_json.workspace = true
> @@ -31,10 +33,11 @@ proxmox-time.workspace = true
> proxmox-uuid = { workspace = true, features = ["serde"] }
>
> [features]
> -default = ["sendmail", "gotify", "smtp"]
> +default = ["sendmail", "gotify", "smtp", "webhook"]
> mail-forwarder = ["dep:mail-parser", "dep:proxmox-sys"]
> -sendmail = ["dep:proxmox-sys"]
> +sendmail = ["dep:proxmox-sys", "dep:base64"]
> gotify = ["dep:proxmox-http"]
> pve-context = ["dep:proxmox-sys"]
> pbs-context = ["dep:proxmox-sys"]
> smtp = ["dep:lettre"]
> +webhook = ["dep:base64", "dep:http", "dep:percent-encoding", "dep:proxmox-http"]
> diff --git a/proxmox-notify/src/config.rs b/proxmox-notify/src/config.rs
> index 789c4a7d..4d0b53f7 100644
> --- a/proxmox-notify/src/config.rs
> +++ b/proxmox-notify/src/config.rs
> @@ -57,6 +57,17 @@ fn config_init() -> SectionConfig {
> GOTIFY_SCHEMA,
> ));
> }
> + #[cfg(feature = "webhook")]
> + {
> + use crate::endpoints::webhook::{WebhookConfig, WEBHOOK_TYPENAME};
> +
> + const WEBHOOK_SCHEMA: &ObjectSchema = WebhookConfig::API_SCHEMA.unwrap_object_schema();
> + config.register_plugin(SectionConfigPlugin::new(
> + WEBHOOK_TYPENAME.to_string(),
> + Some(String::from("name")),
> + WEBHOOK_SCHEMA,
> + ));
> + }
>
> const MATCHER_SCHEMA: &ObjectSchema = MatcherConfig::API_SCHEMA.unwrap_object_schema();
> config.register_plugin(SectionConfigPlugin::new(
> @@ -110,6 +121,18 @@ fn private_config_init() -> SectionConfig {
> ));
> }
>
> + #[cfg(feature = "webhook")]
> + {
> + use crate::endpoints::webhook::{WebhookPrivateConfig, WEBHOOK_TYPENAME};
> +
> + const WEBHOOK_SCHEMA: &ObjectSchema =
> + WebhookPrivateConfig::API_SCHEMA.unwrap_object_schema();
> + config.register_plugin(SectionConfigPlugin::new(
> + WEBHOOK_TYPENAME.to_string(),
> + Some(String::from("name")),
> + WEBHOOK_SCHEMA,
> + ));
> + }
> config
> }
>
> diff --git a/proxmox-notify/src/endpoints/mod.rs b/proxmox-notify/src/endpoints/mod.rs
> index 97f79fcc..f20bee21 100644
> --- a/proxmox-notify/src/endpoints/mod.rs
> +++ b/proxmox-notify/src/endpoints/mod.rs
> @@ -4,5 +4,7 @@ pub mod gotify;
> pub mod sendmail;
> #[cfg(feature = "smtp")]
> pub mod smtp;
> +#[cfg(feature = "webhook")]
> +pub mod webhook;
>
> mod common;
> diff --git a/proxmox-notify/src/endpoints/webhook.rs b/proxmox-notify/src/endpoints/webhook.rs
> new file mode 100644
> index 00000000..7e976f6b
> --- /dev/null
> +++ b/proxmox-notify/src/endpoints/webhook.rs
> @@ -0,0 +1,509 @@
> +use handlebars::{
> + Context as HandlebarsContext, Handlebars, Helper, HelperResult, Output, RenderContext,
> + RenderError as HandlebarsRenderError,
> +};
> +use http::Request;
> +use percent_encoding::AsciiSet;
> +use proxmox_schema::property_string::PropertyString;
> +use serde::{Deserialize, Serialize};
> +use serde_json::{json, Map, Value};
> +
> +use proxmox_http::client::sync::Client;
> +use proxmox_http::{HttpClient, HttpOptions, ProxyConfig};
> +use proxmox_schema::api_types::COMMENT_SCHEMA;
> +use proxmox_schema::{api, ApiStringFormat, ApiType, Schema, StringSchema, Updater};
> +
> +use crate::context::context;
> +use crate::renderer::TemplateType;
> +use crate::schema::ENTITY_NAME_SCHEMA;
> +use crate::{renderer, Content, Endpoint, Error, Notification, Origin};
> +
> +pub(crate) const WEBHOOK_TYPENAME: &str = "webhook";
> +
> +#[api]
> +#[derive(Serialize, Deserialize, Clone, Copy, Default)]
> +#[serde(rename_all = "kebab-case")]
> +/// HTTP Method to use
> +pub enum HttpMethod {
> + /// HTTP POST
> + #[default]
> + Post,
> + /// HTTP PUT
> + Put,
> + /// HTTP GET
> + Get,
> +}
> +
> +// We only ever need a &str, so we rather implement this
> +// instead of Display.
> +impl From<HttpMethod> for &str {
> + fn from(value: HttpMethod) -> Self {
> + match value {
> + HttpMethod::Post => "POST",
> + HttpMethod::Put => "PUT",
> + HttpMethod::Get => "GET",
> + }
> + }
> +}
> +
> +#[api(
> + properties: {
> + name: {
> + schema: ENTITY_NAME_SCHEMA,
> + },
> + comment: {
> + optional: true,
> + schema: COMMENT_SCHEMA,
> + },
> + header: {
> + type: Array,
> + items: {
> + schema: KEY_AND_BASE64_VALUE_SCHEMA,
> + },
> + optional: true,
> + },
> + secret: {
> + type: Array,
> + items: {
> + schema: KEY_AND_BASE64_VALUE_SCHEMA,
> + },
> + optional: true,
> + },
> + }
> +)]
> +#[derive(Serialize, Deserialize, Updater, Default, Clone)]
> +#[serde(rename_all = "kebab-case")]
> +/// Config for Webhook notification endpoints
> +pub struct WebhookConfig {
> + /// Name of the endpoint.
> + #[updater(skip)]
> + pub name: String,
> +
> + pub method: HttpMethod,
> +
> + /// Webhook URL.
> + pub url: String,
> + /// Array of HTTP headers. Each entry is a property string with a name and a value.
> + /// The value property contains the header in base64 encoding.
> + #[serde(default, skip_serializing_if = "Vec::is_empty")]
> + #[updater(serde(skip_serializing_if = "Option::is_none"))]
> + pub header: Vec<PropertyString<KeyAndBase64Val>>,
> + /// Body.
> + #[serde(skip_serializing_if = "Option::is_none")]
> + pub body: Option<String>,
> +
> + /// Comment.
> + #[serde(skip_serializing_if = "Option::is_none")]
> + pub comment: Option<String>,
> + /// Disable this target.
> + #[serde(skip_serializing_if = "Option::is_none")]
> + pub disable: Option<bool>,
> + /// Origin of this config entry.
> + #[serde(skip_serializing_if = "Option::is_none")]
> + #[updater(skip)]
> + pub origin: Option<Origin>,
> + /// Array of secrets. Each entry is a property string with a name and an optional value.
> + /// The value property contains the secret in base64 encoding.
> + /// For any API endpoints returning the endpoint config,
> + /// only the secret name but not the value will be returned.
> + /// When updating the config, also send all secrest that you want
> + /// to keep, setting only the name but not the value.
> + #[serde(default, skip_serializing_if = "Vec::is_empty")]
> + #[updater(serde(skip_serializing_if = "Option::is_none"))]
> + pub secret: Vec<PropertyString<KeyAndBase64Val>>,
> +}
> +
> +#[api(
> + properties: {
> + name: {
> + schema: ENTITY_NAME_SCHEMA,
> + },
> + secret: {
> + type: Array,
> + items: {
> + schema: KEY_AND_BASE64_VALUE_SCHEMA,
> + },
> + optional: true,
> + },
> + }
> +)]
> +#[derive(Serialize, Deserialize, Clone, Updater, Default)]
> +#[serde(rename_all = "kebab-case")]
> +/// Private configuration for Webhook notification endpoints.
> +/// This config will be saved to a separate configuration file with stricter
> +/// permissions (root:root 0600)
> +pub struct WebhookPrivateConfig {
> + /// Name of the endpoint
> + #[updater(skip)]
> + pub name: String,
> +
> + #[serde(default, skip_serializing_if = "Vec::is_empty")]
> + #[updater(serde(skip_serializing_if = "Option::is_none"))]
> + /// Array of secrets. Each entry is a property string with a name,
> + /// and a value property. The value property contains the secret
> + /// in base64 encoding.
> + pub secret: Vec<PropertyString<KeyAndBase64Val>>,
> +}
> +
> +/// A Webhook notification endpoint.
> +pub struct WebhookEndpoint {
> + pub config: WebhookConfig,
> + pub private_config: WebhookPrivateConfig,
> +}
> +
> +#[api]
> +#[derive(Serialize, Deserialize)]
> +#[serde(rename_all = "kebab-case")]
> +pub enum DeleteableWebhookProperty {
> + /// Delete `comment`
> + Comment,
> + /// Delete `disable`
> + Disable,
> + /// Delete `header`
> + Header,
> + /// Delete `body`
> + Body,
> + /// Delete `secret`
> + Secret,
> +}
> +
> +#[api]
> +#[derive(Serialize, Deserialize, Debug, Default, Clone)]
> +/// Datatype used to represent key-value pairs, the value
> +/// being encoded in base64.
> +pub struct KeyAndBase64Val {
> + /// Name
> + pub(crate) name: String,
> + /// Base64 encoded value
> + #[serde(skip_serializing_if = "Option::is_none")]
> + pub(crate) value: Option<String>,
> +}
> +
> +impl KeyAndBase64Val {
> + #[cfg(test)]
> + pub(crate) fn new_with_plain_value(name: &str, value: &str) -> Self {
> + let value = base64::encode(value);
> +
> + Self {
> + name: name.into(),
> + value: Some(value),
> + }
> + }
> +
> + pub(crate) fn decode_value(&self) -> Result<String, Error> {
> + let value = self.value.as_deref().unwrap_or_default();
> + let bytes = base64::decode(value).map_err(|_| {
> + Error::Generic(format!(
> + "could not decode base64 value with name '{}'",
> + self.name
> + ))
> + })?;
> + let value = String::from_utf8(bytes).map_err(|_| {
> + Error::Generic(format!(
> + "could not decode UTF8 string from base64, name={}",
> + self.name
> + ))
> + })?;
> +
> + Ok(value)
> + }
> +}
> +
> +pub const KEY_AND_BASE64_VALUE_SCHEMA: Schema =
> + StringSchema::new("String schema for pairs of keys and base64 encoded values")
> + .format(&ApiStringFormat::PropertyString(
> + &KeyAndBase64Val::API_SCHEMA,
> + ))
> + .schema();
> +
> +impl Endpoint for WebhookEndpoint {
> + fn send(&self, notification: &Notification) -> Result<(), Error> {
> + let request = self.build_request(notification)?;
> +
> + self.create_client()?
> + .request(request)
> + .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?;
> +
> + Ok(())
> + }
> +
> + fn name(&self) -> &str {
> + &self.config.name
> + }
> +
> + /// Check if the endpoint is disabled
> + fn disabled(&self) -> bool {
> + self.config.disable.unwrap_or_default()
> + }
> +}
> +
> +impl WebhookEndpoint {
> + fn create_client(&self) -> Result<Client, Error> {
> + let proxy_config = context()
> + .http_proxy_config()
> + .map(|url| ProxyConfig::parse_proxy_url(&url))
> + .transpose()
> + .map_err(|err| Error::NotifyFailed(self.name().to_string(), err.into()))?;
> +
> + let options = HttpOptions {
> + proxy_config,
> + ..Default::default()
> + };
> +
> + Ok(Client::new(options))
> + }
> +
> + fn build_request(&self, notification: &Notification) -> Result<Request<String>, Error> {
> + let (title, message) = match ¬ification.content {
> + Content::Template {
> + template_name,
> + data,
> + } => {
> + let rendered_title =
> + renderer::render_template(TemplateType::Subject, template_name, data)?;
> + let rendered_message =
> + renderer::render_template(TemplateType::PlaintextBody, template_name, data)?;
> +
> + (rendered_title, rendered_message)
> + }
> + #[cfg(feature = "mail-forwarder")]
> + Content::ForwardedMail { title, body, .. } => (title.clone(), body.clone()),
> + };
> +
> + let mut fields = Map::new();
> +
> + for (field_name, field_value) in ¬ification.metadata.additional_fields {
> + fields.insert(field_name.clone(), Value::String(field_value.to_string()));
> + }
> +
> + let mut secrets = Map::new();
> +
> + for secret in &self.private_config.secret {
> + let value = secret.decode_value()?;
> + secrets.insert(secret.name.clone(), Value::String(value));
> + }
> +
> + let data = json!({
> + "title": &title,
> + "message": &message,
> + "severity": notification.metadata.severity,
> + "timestamp": notification.metadata.timestamp,
> + "fields": fields,
> + "secrets": secrets,
> + });
> +
> + let handlebars = setup_handlebars();
> + let body_template = self.base_64_decode(self.config.body.as_deref().unwrap_or_default())?;
> +
> + let body = handlebars
> + .render_template(&body_template, &data)
> + .map_err(|err| {
> + // TODO: Cleanup error types, they have become a bit messy.
> + // No user of the notify crate distinguish between the error types any way, so
> + // we can refactor without any issues....
> + Error::Generic(format!("failed to render webhook body: {err}"))
I'm curious, how would you clean up the error types in particular?
> + })?;
> +
> + let url = handlebars
> + .render_template(&self.config.url, &data)
> + .map_err(|err| Error::Generic(format!("failed to render webhook url: {err}")))?;
> +
> + let method: &str = self.config.method.into();
> + let mut builder = http::Request::builder().uri(url).method(method);
> +
> + for header in &self.config.header {
> + let value = header.decode_value()?;
> +
> + let value = handlebars.render_template(&value, &data).map_err(|err| {
> + Error::Generic(format!(
> + "failed to render header value template: {value}: {err}"
> + ))
> + })?;
> +
> + builder = builder.header(header.name.clone(), value);
> + }
> +
> + let request = builder
> + .body(body)
> + .map_err(|err| Error::Generic(format!("failed to build http request: {err}")))?;
> +
> + Ok(request)
> + }
> +
> + fn base_64_decode(&self, s: &str) -> Result<String, Error> {
> + // Also here, TODO: revisit Error variants for the *whole* crate.
> + let s = base64::decode(s)
> + .map_err(|err| Error::Generic(format!("could not decode base64 value: {err}")))?;
> +
> + String::from_utf8(s).map_err(|err| {
> + Error::Generic(format!(
> + "base64 encoded value did not contain valid utf8: {err}"
> + ))
> + })
> + }
> +}
> +
> +fn setup_handlebars() -> Handlebars<'static> {
> + let mut handlebars = Handlebars::new();
> +
> + handlebars.register_helper("url-encode", Box::new(handlebars_percent_encode));
> + handlebars.register_helper("json", Box::new(handlebars_json));
> + handlebars.register_helper("escape", Box::new(handlebars_escape));
> +
> + // There is no escape.
> + handlebars.register_escape_fn(handlebars::no_escape);
> +
> + handlebars
> +}
> +
> +fn handlebars_percent_encode(
> + h: &Helper,
> + _: &Handlebars,
> + _: &HandlebarsContext,
> + _rc: &mut RenderContext,
> + out: &mut dyn Output,
> +) -> HelperResult {
> + let param0 = h
> + .param(0)
> + .and_then(|v| v.value().as_str())
> + .ok_or_else(|| HandlebarsRenderError::new("url-encode: missing parameter"))?;
> +
> + // See https://developer.mozilla.org/en-US/docs/Glossary/Percent-encoding
> + const FRAGMENT: &AsciiSet = &percent_encoding::CONTROLS
> + .add(b':')
> + .add(b'/')
> + .add(b'?')
> + .add(b'#')
> + .add(b'[')
> + .add(b']')
> + .add(b'@')
> + .add(b'!')
> + .add(b'$')
> + .add(b'&')
> + .add(b'\'')
> + .add(b'(')
> + .add(b')')
> + .add(b'*')
> + .add(b'+')
> + .add(b',')
> + .add(b';')
> + .add(b'=')
> + .add(b'%')
> + .add(b' ');
> + let a = percent_encoding::utf8_percent_encode(param0, FRAGMENT);
> +
> + out.write(&a.to_string())?;
> +
> + Ok(())
> +}
> +
> +fn handlebars_json(
> + h: &Helper,
> + _: &Handlebars,
> + _: &HandlebarsContext,
> + _rc: &mut RenderContext,
> + out: &mut dyn Output,
> +) -> HelperResult {
> + let param0 = h
> + .param(0)
> + .map(|v| v.value())
> + .ok_or_else(|| HandlebarsRenderError::new("json: missing parameter"))?;
> +
> + let json = serde_json::to_string(param0)?;
> + out.write(&json)?;
> +
> + Ok(())
> +}
> +
> +fn handlebars_escape(
> + h: &Helper,
> + _: &Handlebars,
> + _: &HandlebarsContext,
> + _rc: &mut RenderContext,
> + out: &mut dyn Output,
> +) -> HelperResult {
> + let text = h
> + .param(0)
> + .and_then(|v| v.value().as_str())
> + .ok_or_else(|| HandlebarsRenderError::new("escape: missing text parameter"))?;
> +
> + let val = Value::String(text.to_string());
> + let json = serde_json::to_string(&val)?;
> + out.write(&json[1..json.len() - 1])?;
> +
> + Ok(())
> +}
> +
> +#[cfg(test)]
> +mod tests {
> + use std::collections::HashMap;
> +
> + use super::*;
> + use crate::Severity;
> +
> +
> +
> + #[test]
> + fn test_build_request() -> Result<(), Error> {
> + let data = HashMap::from_iter([
> + ("hello".into(), "hello world".into()),
> + ("test".into(), "escaped\nstring".into()),
> + ]);
> +
> + let body_template = r#"
> +{{ fields.test }}
> +{{ escape fields.test }}
> +
> +{{ json fields }}
> +{{ json fields.hello }}
> +
> +{{ url-encode fields.hello }}
> +
> +{{ json severity }}
> +
> +"#;
> +
> + let expected_body = r#"
> +escaped
> +string
> +escaped\nstring
> +
> +{"hello":"hello world","test":"escaped\nstring"}
> +"hello world"
> +
> +hello%20world
> +
> +"info"
> +
> +"#;
> +
> + let endpoint = WebhookEndpoint {
> + config: WebhookConfig {
> + name: "test".into(),
> + method: HttpMethod::Post,
> + url: "http://localhost/{{ url-encode fields.hello }}".into(),
> + header: vec![
> + KeyAndBase64Val::new_with_plain_value("X-Severity", "{{ severity }}").into(),
> + ],
> + body: Some(base64::encode(body_template)),
> + ..Default::default()
> + },
> + private_config: WebhookPrivateConfig {
> + name: "test".into(),
> + ..Default::default()
> + },
> + };
> +
> + let notification = Notification::from_template(Severity::Info, "foo", json!({}), data);
> +
> + let request = endpoint.build_request(¬ification)?;
> +
> + assert_eq!(request.uri(), "http://localhost/hello%20world");
> + assert_eq!(request.body(), expected_body);
> + assert_eq!(request.method(), "POST");
> +
> + assert_eq!(request.headers().get("X-Severity").unwrap(), "info");
> +
> + Ok(())
> + }
> +}
> diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs
> index 910dfa06..ebaba119 100644
> --- a/proxmox-notify/src/lib.rs
> +++ b/proxmox-notify/src/lib.rs
> @@ -499,6 +499,23 @@ impl Bus {
> );
> }
>
> + #[cfg(feature = "webhook")]
> + {
> + use endpoints::webhook::WEBHOOK_TYPENAME;
> + use endpoints::webhook::{WebhookConfig, WebhookEndpoint, WebhookPrivateConfig};
> + endpoints.extend(
> + parse_endpoints_with_private_config!(
> + config,
> + WebhookConfig,
> + WebhookPrivateConfig,
> + WebhookEndpoint,
> + WEBHOOK_TYPENAME
> + )?
> + .into_iter()
> + .map(|e| (e.name().into(), e)),
> + );
> + }
> +
> let matchers = config
> .config
> .convert_to_typed_array(MATCHER_TYPENAME)
More information about the pve-devel
mailing list