[pbs-devel] [RFC proxmox 2/2] proxmox-log: added tracing infra
Dominik Csapak
d.csapak at proxmox.com
Wed Oct 18 15:26:52 CEST 2023
i just glanced over the code but a few comments inline
On 10/11/23 16:01, Gabriel Goller wrote:
[..snip..]
> diff --git a/proxmox-log/src/file_layer.rs b/proxmox-log/src/file_layer.rs
> new file mode 100644
> index 0000000..1a167f4
> --- /dev/null
> +++ b/proxmox-log/src/file_layer.rs
> @@ -0,0 +1,89 @@
> +use std::cell::RefCell;
> +use tracing::error;
> +use tracing::field::Field;
> +use tracing::field::Visit;
> +use tracing::Event;
> +use tracing::Level;
> +use tracing::Subscriber;
> +use tracing_subscriber::layer::Context;
> +use tracing_subscriber::Layer;
> +
> +use crate::FileLogger;
> +
> +tokio::task_local! {
> + pub static LOGGER: RefCell<Option<FileLogger>>;
> + pub static WARN_COUNTER: RefCell<Option<u64>>;
why are these wrapped in options? with 'try_with' you get an error
anyway when they don't exists and i did not see a place
where you'd set them to None?
> +}
> +
> +pub struct FilelogLayer {}
> +
> +impl FilelogLayer {
> + pub fn new() -> Self {
> + Self {}
> + }
> +}
> +
> +impl<S: Subscriber> Layer<S> for FilelogLayer {
> + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
> + let mut buf = String::new();
> +
> + event.record(&mut EventVisitor::new(&mut buf));
> +
> + let logger_exists = LOGGER.try_with(|logger| {
> + let mut l = logger.borrow_mut();
> + let log = l.as_mut();
> + if let Some(l) = log {
> + log_to_file(l, event.metadata().level(), buf);
> + } else {
> + error!("FileLogger not found");
> + }
> + });
> + if let Err(e) = logger_exists {
> + error!(
> + "Error getting the logger from the task-local storage: {}",
> + e
> + )
> + }
this would be much shorter if it wasn't for the Option<>
> + }
> +}
> +
> +fn log_to_file(logger: &mut FileLogger, level: &Level, buf: String) {
> + match level {
> + &Level::ERROR | &Level::WARN => {
> + let counter_exists = WARN_COUNTER.try_with(|counter| > + counter.replace_with(|c| {
> + if let Some(v) = c {
> + Some(v.to_owned() + 1)
> + } else {
> + None
> + }
> + });
two things here:
* do we really want to increase the warn counter for error too?
(i can't remember what we do now)
normally we abort on an error anyway, so that does not make sense
or if we now can log an error without aborting, we'd probably want
to have a separate error count?
* clippy says this is a manually implemented map:
counter.replace_with(|c| c.as_mut().map(|v| v.to_owned() + 1));
> + logger.log(buf.clone());
> + });
> + if let Err(e) = counter_exists {
> + error!("Error getting the warning_counter for the logger: {}", e)
> + }
> + }
> + &Level::INFO => logger.log(buf.clone()),
this clone is redundant (thanks clippy ;) )
> + &Level::DEBUG => logger.log(format!("DEBUG: {}", buf)),
> + &Level::TRACE => logger.log(format!("TRACE: {}", buf)),
> + };
> +}
> +
> +struct EventVisitor<'a> {
> + buf: &'a mut String,
> +}
> +
> +impl<'a> EventVisitor<'a> {
> + fn new(buf: &'a mut String) -> Self {
> + Self { buf }
> + }
> +}
> +
> +impl Visit for EventVisitor<'_> {
> + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
> + if field.name() == "message" {
> + self.buf.push_str(&format!("{:?}", value));
> + }
> + }
> +}
> diff --git a/proxmox-rest-server/src/file_logger.rs b/proxmox-log/src/file_logger.rs
> similarity index 98%
> rename from proxmox-rest-server/src/file_logger.rs
> rename to proxmox-log/src/file_logger.rs
> index 2bb1fac..c7e1d64 100644
> --- a/proxmox-rest-server/src/file_logger.rs
> +++ b/proxmox-log/src/file_logger.rs
> @@ -30,7 +30,7 @@ pub struct FileLogOptions {
> /// #### Example:
> /// ```
> /// # use anyhow::{bail, format_err, Error};
> -/// use proxmox_rest_server::{flog, FileLogger, FileLogOptions};
> +/// use proxmox_log::{flog, FileLogger, FileLogOptions};
> ///
> /// # std::fs::remove_file("test.log");
> /// let options = FileLogOptions {
> diff --git a/proxmox-log/src/lib.rs b/proxmox-log/src/lib.rs
> new file mode 100644
> index 0000000..0c101b7
> --- /dev/null
> +++ b/proxmox-log/src/lib.rs
> @@ -0,0 +1,36 @@
> +//! # Proxmox logging infrastructure
> +use crate::syslog_layer::SyslogLayer;
> +use file_layer::FilelogLayer;
> +use std::env;
> +use tracing_log::LogTracer;
> +use tracing_subscriber::filter::{filter_fn, LevelFilter};
> +use tracing_subscriber::prelude::*;
> +
> +pub mod file_layer;
> +mod file_logger;
> +pub use file_logger::{FileLogOptions, FileLogger};
> +
> +mod syslog_layer;
> +
> +pub fn init_logger(
> + env_var_name: &str,
> + default_log_level: LevelFilter,
> + application_name: &str,
> +) -> Result<(), anyhow::Error> {
> + let mut log_level = default_log_level;
> + if let Ok(v) = env::var(env_var_name) {
> + if let Ok(l) = v.parse::<LevelFilter>() {
> + log_level = l;
> + }
> + }
> + let registry = tracing_subscriber::registry()
> + .with(FilelogLayer::new().with_filter(filter_fn(|metadata| {
> + metadata.fields().field("tasklog").is_some()
you give 'tasklog = true' but only check here for is_some, wouldn't
it be better to do
metadata.fields().field("tasklog") == Some("true")
?
that way a
'tasklog = false' wouldn't (confusingly) log to the task log
> + })))
> + .with(SyslogLayer::new(application_name.to_string()).with_filter(log_level));
> +
> + tracing::subscriber::set_global_default(registry)?;
> +
> + LogTracer::init()?;
> + Ok(())
> +}
> diff --git a/proxmox-log/src/syslog_layer.rs b/proxmox-log/src/syslog_layer.rs
> new file mode 100644
> index 0000000..50ea145
> --- /dev/null
> +++ b/proxmox-log/src/syslog_layer.rs
> @@ -0,0 +1,78 @@
> +use std::fmt;
> +use std::sync::Arc;
> +use std::sync::Mutex;
> +use syslog::Formatter3164;
> +use syslog::Logger;
> +use syslog::LoggerBackend;
> +use tracing::field::Field;
> +use tracing::field::Visit;
> +use tracing::Event;
> +use tracing::Level;
> +use tracing::Subscriber;
> +use tracing_subscriber::layer::Context;
> +use tracing_subscriber::Layer;
> +
> +pub struct SyslogLayer {
> + logger: Arc<Mutex<Logger<LoggerBackend, Formatter3164>>>,
> +}
> +
> +impl SyslogLayer {
> + pub fn new(application_name: String) -> Self {
> + let mut formatter = Formatter3164::default();
> + formatter.facility = syslog::Facility::LOG_DAEMON;
> + formatter.process = application_name;
> +
> + // unwrap because if the (syslog) logging is not working, we shut down
> + let backend = syslog::unix(formatter.clone())
> + .map(|logger: Logger<LoggerBackend, Formatter3164>| logger.backend)
> + .map_err(|e| {
> + anyhow::Error::new(std::io::Error::new(
> + std::io::ErrorKind::Other,
> + e.description(),
> + ))
> + })
> + .unwrap();
> +
> + let logger = Arc::new(Mutex::new(Logger { formatter, backend }));
> +
> + Self { logger }
> + }
> +}
> +
> +impl<S: Subscriber> Layer<S> for SyslogLayer {
> + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
> + let mut buf = String::new();
> +
> + event.record(&mut EventVisitor::new(&mut buf));
> +
> + let mut logger = self.logger.lock().unwrap();
> + let succes = match event.metadata().level() {
> + &Level::ERROR => logger.err(buf),
> + &Level::WARN => logger.warning(buf),
> + &Level::INFO => logger.info(buf),
> + &Level::DEBUG => logger.debug(buf),
> + &Level::TRACE => logger.debug(buf),
> + };
> + if let Err(err) = succes {
> + log::error!("{}", err.description());
> + }
> + }
> +}
> +
> +struct EventVisitor<'a> {
> + buf: &'a mut String,
> +}
> +
> +impl<'a> EventVisitor<'a> {
> + fn new(buf: &'a mut String) -> Self {
> + Self { buf }
> + }
> +}
> +
> +impl Visit for EventVisitor<'_> {
> + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
> + if field.name() == "message" {
> + self.buf.push_str(&format!("{:?}", value));
> + }
> + }
> +}
> diff --git a/proxmox-rest-server/Cargo.toml b/proxmox-rest-server/Cargo.toml
> index 8daf1d3..a444941 100644
> --- a/proxmox-rest-server/Cargo.toml
> +++ b/proxmox-rest-server/Cargo.toml
> @@ -34,6 +34,7 @@ tokio-openssl.workspace = true
> tokio-stream.workspace = true
> tower-service.workspace = true
> url.workspace = true
> +tracing.workspace = true
>
> proxmox-async.workspace = true
> proxmox-compression.workspace = true
> @@ -44,6 +45,7 @@ proxmox-router.workspace = true
> proxmox-schema = { workspace = true, features = [ "api-macro", "upid-api-impl" ] }
> proxmox-sys = { workspace = true, features = [ "logrotate", "timer" ] }
> proxmox-time.workspace = true
> +proxmox-log.workspace = true
>
> [features]
> default = []
> diff --git a/proxmox-rest-server/src/api_config.rs b/proxmox-rest-server/src/api_config.rs
> index ad9a811..cf14c8f 100644
> --- a/proxmox-rest-server/src/api_config.rs
> +++ b/proxmox-rest-server/src/api_config.rs
> @@ -9,11 +9,12 @@ use http::{HeaderMap, Method};
> use hyper::http::request::Parts;
> use hyper::{Body, Response};
>
> +use proxmox_log::{FileLogOptions, FileLogger};
> use proxmox_router::{Router, RpcEnvironmentType, UserInformation};
> use proxmox_sys::fs::{create_path, CreateOptions};
>
> use crate::rest::Handler;
> -use crate::{CommandSocket, FileLogOptions, FileLogger, RestEnvironment};
> +use crate::{CommandSocket, RestEnvironment};
>
> /// REST server configuration
> pub struct ApiConfig {
> diff --git a/proxmox-rest-server/src/lib.rs b/proxmox-rest-server/src/lib.rs
> index bc5be01..9c16065 100644
> --- a/proxmox-rest-server/src/lib.rs
> +++ b/proxmox-rest-server/src/lib.rs
> @@ -41,9 +41,6 @@ pub use state::*;
> mod command_socket;
> pub use command_socket::*;
>
> -mod file_logger;
> -pub use file_logger::{FileLogOptions, FileLogger};
> -
> mod api_config;
> pub use api_config::{ApiConfig, AuthError, AuthHandler, IndexHandler};
>
> diff --git a/proxmox-rest-server/src/rest.rs b/proxmox-rest-server/src/rest.rs
> index 2ccd4d5..9ad01a5 100644
> --- a/proxmox-rest-server/src/rest.rs
> +++ b/proxmox-rest-server/src/rest.rs
> @@ -31,10 +31,10 @@ use proxmox_schema::{ObjectSchemaType, ParameterSchema};
>
> use proxmox_async::stream::AsyncReaderStream;
> use proxmox_compression::{DeflateEncoder, Level};
> +use proxmox_log::FileLogger;
>
> use crate::{
> - formatter::*, normalize_path, ApiConfig, AuthError, CompressionMethod, FileLogger,
> - RestEnvironment,
> + formatter::*, normalize_path, ApiConfig, AuthError, CompressionMethod, RestEnvironment,
> };
>
> extern "C" {
> diff --git a/proxmox-rest-server/src/worker_task.rs b/proxmox-rest-server/src/worker_task.rs
> index 54b6bc2..f7b8396 100644
> --- a/proxmox-rest-server/src/worker_task.rs
> +++ b/proxmox-rest-server/src/worker_task.rs
> @@ -1,3 +1,4 @@
> +use std::cell::RefCell;
> use std::collections::{HashMap, VecDeque};
> use std::fs::File;
> use std::io::{BufRead, BufReader, Read, Write};
> @@ -12,21 +13,23 @@ use futures::*;
> use lazy_static::lazy_static;
> use nix::fcntl::OFlag;
> use once_cell::sync::OnceCell;
> +use proxmox_log::file_layer::{LOGGER, WARN_COUNTER};
> use serde::{Deserialize, Serialize};
> use serde_json::{json, Value};
> use tokio::signal::unix::SignalKind;
> use tokio::sync::oneshot;
> +use tracing::{info, warn};
>
> use proxmox_lang::try_block;
> +use proxmox_log::{FileLogOptions, FileLogger};
> use proxmox_schema::upid::UPID;
> use proxmox_sys::fs::{atomic_open_or_create_file, create_path, replace_file, CreateOptions};
> use proxmox_sys::linux::procfs;
> -use proxmox_sys::task_warn;
>
> use proxmox_sys::logrotate::{LogRotate, LogRotateFiles};
> use proxmox_sys::WorkerTaskContext;
>
> -use crate::{CommandSocket, FileLogOptions, FileLogger};
> +use crate::CommandSocket;
>
> struct TaskListLockGuard(File);
>
> @@ -274,7 +277,7 @@ pub fn rotate_task_log_archive(
>
> /// removes all task logs that are older than the oldest task entry in the
> /// task archive
> -pub fn cleanup_old_tasks(worker: &dyn WorkerTaskContext, compressed: bool) -> Result<(), Error> {
> +pub fn cleanup_old_tasks(compressed: bool) -> Result<(), Error> {
> let setup = worker_task_setup()?;
>
> let _lock = setup.lock_task_list_files(true)?;
> @@ -312,7 +315,10 @@ pub fn cleanup_old_tasks(worker: &dyn WorkerTaskContext, compressed: bool) -> Re
> Ok(files) => files,
> Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
> Err(err) => {
> - task_warn!(worker, "could not check task logs in '{:02X}': {}", i, err);
> + warn!(
> + tasklog = true,
> + "could not check task logs in '{:02X}': {}", i, err
> + );
> continue;
> }
> };
> @@ -320,11 +326,9 @@ pub fn cleanup_old_tasks(worker: &dyn WorkerTaskContext, compressed: bool) -> Re
> let file = match file {
> Ok(file) => file,
> Err(err) => {
> - task_warn!(
> - worker,
> - "could not check some task log in '{:02X}': {}",
> - i,
> - err
> + warn!(
> + tasklog = true,
> + "could not check some task log in '{:02X}': {}", i, err
> );
> continue;
> }
> @@ -334,7 +338,10 @@ pub fn cleanup_old_tasks(worker: &dyn WorkerTaskContext, compressed: bool) -> Re
> let modified = match get_modified(file) {
> Ok(modified) => modified,
> Err(err) => {
> - task_warn!(worker, "error getting mtime for '{:?}': {}", path, err);
> + warn!(
> + tasklog = true,
> + "error getting mtime for '{:?}': {}", path, err
> + );
> continue;
> }
> };
> @@ -344,7 +351,10 @@ pub fn cleanup_old_tasks(worker: &dyn WorkerTaskContext, compressed: bool) -> Re
> Ok(()) => {}
> Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
> Err(err) => {
> - task_warn!(worker, "could not remove file '{:?}': {}", path, err)
> + warn!(
> + tasklog = true,
> + "could not remove file '{:?}': {}", path, err
> + )
> }
> }
> }
> @@ -796,9 +806,7 @@ impl std::fmt::Display for WorkerTask {
> }
>
> struct WorkerTaskData {
> - logger: FileLogger,
> progress: f64, // 0..1
> - warn_count: u64,
> pub abort_listeners: Vec<oneshot::Sender<()>>,
> }
>
> @@ -808,7 +816,7 @@ impl WorkerTask {
> worker_id: Option<String>,
> auth_id: String,
> to_stdout: bool,
> - ) -> Result<Arc<Self>, Error> {
> + ) -> Result<(Arc<Self>, FileLogger), Error> {
> let setup = worker_task_setup()?;
>
> let upid = UPID::new(worker_type, worker_id, auth_id)?;
> @@ -842,9 +850,7 @@ impl WorkerTask {
> upid: upid.clone(),
> abort_requested: AtomicBool::new(false),
> data: Mutex::new(WorkerTaskData {
> - logger,
> progress: 0.0,
> - warn_count: 0,
> abort_listeners: vec![],
> }),
> });
> @@ -858,7 +864,7 @@ impl WorkerTask {
>
> setup.update_active_workers(Some(&upid))?;
>
> - Ok(worker)
> + Ok((worker, logger))
> }
>
> /// Spawn a new tokio task/future.
> @@ -873,12 +879,21 @@ impl WorkerTask {
> F: Send + 'static + FnOnce(Arc<WorkerTask>) -> T,
> T: Send + 'static + Future<Output = Result<(), Error>>,
> {
> - let worker = WorkerTask::new(worker_type, worker_id, auth_id, to_stdout)?;
> + let (worker, logger) = WorkerTask::new(worker_type, worker_id, auth_id, to_stdout)?;
> let upid_str = worker.upid.to_string();
> let f = f(worker.clone());
> +
> + let logger = RefCell::new(Some(logger));
> + let counter = RefCell::new(Some(0));
> tokio::spawn(async move {
> - let result = f.await;
> - worker.log_result(&result);
> + LOGGER.scope(logger, async move {
> + WARN_COUNTER.scope(counter, async move {
> + let result = f.await;
> + worker.log_result(&result);
> + })
> + .await;
> + })
> + .await;
> });
>
> Ok(upid_str)
> @@ -895,22 +910,27 @@ impl WorkerTask {
> where
> F: Send + UnwindSafe + 'static + FnOnce(Arc<WorkerTask>) -> Result<(), Error>,
> {
> - let worker = WorkerTask::new(worker_type, worker_id, auth_id, to_stdout)?;
> + let (worker, logger) = WorkerTask::new(worker_type, worker_id, auth_id, to_stdout)?;
> let upid_str = worker.upid.to_string();
>
> let _child = std::thread::Builder::new()
> .name(upid_str.clone())
> .spawn(move || {
> - let worker1 = worker.clone();
> - let result = match std::panic::catch_unwind(move || f(worker1)) {
> - Ok(r) => r,
> - Err(panic) => match panic.downcast::<&str>() {
> - Ok(panic_msg) => Err(format_err!("worker panicked: {}", panic_msg)),
> - Err(_) => Err(format_err!("worker panicked: unknown type.")),
> - },
> - };
> + LOGGER.sync_scope(RefCell::new(Some(logger)), || {
> + WARN_COUNTER.sync_scope(RefCell::new(Some(0)), || {
> + let worker1 = worker.clone();
> +
> + let result = match std::panic::catch_unwind(move || f(worker1)) {
> + Ok(r) => r,
> + Err(panic) => match panic.downcast::<&str>() {
> + Ok(panic_msg) => Err(format_err!("worker panicked: {}", panic_msg)),
> + Err(_) => Err(format_err!("worker panicked: unknown type.")),
> + },
> + };
>
> - worker.log_result(&result);
> + worker.log_result(&result);
> + });
> + });
> });
>
> Ok(upid_str)
> @@ -918,7 +938,13 @@ impl WorkerTask {
>
> /// create state from self and a result
> pub fn create_state(&self, result: &Result<(), Error>) -> TaskState {
> - let warn_count = self.data.lock().unwrap().warn_count;
> + let mut warn_count: u64 = 0;
> +
> + let _ = WARN_COUNTER.try_with(|counter| {
> + if let Some(v) = counter.borrow_mut().as_mut() {
> + warn_count = v.to_owned();
> + }
> + });
>
> let endtime = proxmox_time::epoch_i64();
>
> @@ -949,15 +975,7 @@ impl WorkerTask {
>
> /// Log a message.
> pub fn log_message<S: AsRef<str>>(&self, msg: S) {
> - let mut data = self.data.lock().unwrap();
> - data.logger.log(msg);
> - }
> -
> - /// Log a message as warning.
> - pub fn log_warning<S: AsRef<str>>(&self, msg: S) {
> - let mut data = self.data.lock().unwrap();
> - data.logger.log(format!("WARN: {}", msg.as_ref()));
> - data.warn_count += 1;
> + info!(tasklog = true, "{}", msg.as_ref());
> }
>
> /// Set progress indicator
> @@ -1020,16 +1038,6 @@ impl WorkerTaskContext for WorkerTask {
> fn fail_on_shutdown(&self) -> Result<(), Error> {
> crate::fail_on_shutdown()
> }
> -
> - fn log(&self, level: log::Level, message: &std::fmt::Arguments) {
> - match level {
> - log::Level::Error => self.log_warning(message.to_string()),
> - log::Level::Warn => self.log_warning(message.to_string()),
> - log::Level::Info => self.log_message(message.to_string()),
> - log::Level::Debug => self.log_message(format!("DEBUG: {}", message)),
> - log::Level::Trace => self.log_message(format!("TRACE: {}", message)),
> - }
> - }
> }
>
> /// Wait for a locally spanned worker task
> diff --git a/proxmox-router/Cargo.toml b/proxmox-router/Cargo.toml
> index c53ccd0..2460999 100644
> --- a/proxmox-router/Cargo.toml
> +++ b/proxmox-router/Cargo.toml
> @@ -19,6 +19,8 @@ percent-encoding.workspace = true
> serde_json.workspace = true
> serde.workspace = true
> unicode-width ="0.1.8"
> +syslog = "6"
> +log = "0.4.17"
>
> # cli:
> tokio = { workspace = true, features = [], optional = true }
> diff --git a/proxmox-sys/src/worker_task_context.rs b/proxmox-sys/src/worker_task_context.rs
> index 2c86857..743ae53 100644
> --- a/proxmox-sys/src/worker_task_context.rs
> +++ b/proxmox-sys/src/worker_task_context.rs
> @@ -26,9 +26,6 @@ pub trait WorkerTaskContext: Send + Sync {
> }
> Ok(())
> }
> -
> - /// Create a log message for this task.
> - fn log(&self, level: log::Level, message: &std::fmt::Arguments);
> }
>
> /// Convenience implementation:
> @@ -48,48 +45,4 @@ impl<T: WorkerTaskContext + ?Sized> WorkerTaskContext for std::sync::Arc<T> {
> fn fail_on_shutdown(&self) -> Result<(), Error> {
> <T as WorkerTaskContext>::fail_on_shutdown(self)
> }
> -
> - fn log(&self, level: log::Level, message: &std::fmt::Arguments) {
> - <T as WorkerTaskContext>::log(self, level, message)
> - }
> -}
> -
> -/// Log an error to a [WorkerTaskContext]
> -#[macro_export]
> -macro_rules! task_error {
> - ($task:expr, $($fmt:tt)+) => {{
> - $crate::WorkerTaskContext::log(&*$task, log::Level::Error, &format_args!($($fmt)+))
> - }};
> -}
> -
> -/// Log a warning to a [WorkerTaskContext]
> -#[macro_export]
> -macro_rules! task_warn {
> - ($task:expr, $($fmt:tt)+) => {{
> - $crate::WorkerTaskContext::log(&*$task, log::Level::Warn, &format_args!($($fmt)+))
> - }};
> -}
> -
> -/// Log a message to a [WorkerTaskContext]
> -#[macro_export]
> -macro_rules! task_log {
> - ($task:expr, $($fmt:tt)+) => {{
> - $crate::WorkerTaskContext::log(&*$task, log::Level::Info, &format_args!($($fmt)+))
> - }};
> -}
> -
> -/// Log a debug message to a [WorkerTaskContext]
> -#[macro_export]
> -macro_rules! task_debug {
> - ($task:expr, $($fmt:tt)+) => {{
> - $crate::WorkerTaskContext::log(&*$task, log::Level::Debug, &format_args!($($fmt)+))
> - }};
> -}
> -
> -/// Log a trace message to a [WorkerTaskContext]
> -#[macro_export]
> -macro_rules! task_trace {
> - ($task:expr, $($fmt:tt)+) => {{
> - $crate::WorkerTaskContext::log(&*$task, log::Level::Trace, &format_args!($($fmt)+))
> - }};
> }
More information about the pbs-devel
mailing list