[pbs-devel] [PATCH proxmox-backup] Add .../apt/update API call
Fabian Grünbichler
f.gruenbichler at proxmox.com
Wed Jul 15 15:34:42 CEST 2020
On July 14, 2020 11:13 am, Stefan Reiter wrote:
> Lists all available package updates via libapt-pkg. Output format is
> taken from PVE to enable JS component reuse in the future.
>
> Depends on apt-pkg-native-rs. Changelog-URL detection is inspired by PVE
> perl code (but modified).
>
> list_installed_apt_packages iterates all packages and creates an
> APTUpdateInfo with detailed information for every package matched by the
> given filter Fn.
>
> libapt-pkg has some questionable design choices regarding their use of
> 'iterators', which means quite a bit of nesting sadly...
>
> Signed-off-by: Stefan Reiter <s.reiter at proxmox.com>
> ---
>
> apt-pkg-native-rs requires some custom patches on top of the current upstream to
> be able to access all required information. I sent a PR to upstream, but it
> hasn't been included as of yet.
>
> The package is not mentioned in Cargo.toml, since we don't have an internal
> package for it, but for testing you can include my fork with the patches:
>
> apt-pkg-native = { git = "https://github.com/PiMaker/apt-pkg-native-rs" }
>
> The original is here: https://github.com/FauxFaux/apt-pkg-native-rs
this is now available with your patches in devel, probably warrants a
comment that this is a special patched one when we add it to Cargo.toml
> Also, the changelog URL detection was initially just taken from Perl code, but
> it turns out we have some slightly different information there, so I did my best
> to rewrite it to be accurate. With this implementation I get a "200 OK" on the
> generated changelog URL of all default-installed packages on PBS, except for two
> with recent security updates (as mentioned in the code comment, they don't seem
> to have changelogs at all).
>
> I'll probably take a look at the perl code in the future to see if it can be
> improved as well, some Debian changelogs produce 404 URLs for me there.
>
>
> src/api2/node.rs | 2 +
> src/api2/node/apt.rs | 205 +++++++++++++++++++++++++++++++++++++++++++
> src/api2/types.rs | 27 ++++++
> 3 files changed, 234 insertions(+)
> create mode 100644 src/api2/node/apt.rs
>
> diff --git a/src/api2/node.rs b/src/api2/node.rs
> index 13ff282c..7e70bc04 100644
> --- a/src/api2/node.rs
> +++ b/src/api2/node.rs
> @@ -11,8 +11,10 @@ mod services;
> mod status;
> pub(crate) mod rrd;
> pub mod disks;
> +mod apt;
>
> pub const SUBDIRS: SubdirMap = &[
> + ("apt", &apt::ROUTER),
> ("disks", &disks::ROUTER),
> ("dns", &dns::ROUTER),
> ("journal", &journal::ROUTER),
> diff --git a/src/api2/node/apt.rs b/src/api2/node/apt.rs
> new file mode 100644
> index 00000000..8b1f2ecf
> --- /dev/null
> +++ b/src/api2/node/apt.rs
> @@ -0,0 +1,205 @@
> +use apt_pkg_native::Cache;
> +use anyhow::{Error, bail};
> +use serde_json::{json, Value};
> +
> +use proxmox::{list_subdirs_api_method, const_regex};
> +use proxmox::api::{api, Router, Permission, SubdirMap};
> +
> +use crate::config::acl::PRIV_SYS_AUDIT;
> +use crate::api2::types::{APTUpdateInfo, NODE_SCHEMA};
> +
> +const_regex! {
> + VERSION_EPOCH_REGEX = r"^\d+:";
> + FILENAME_EXTRACT_REGEX = r"^.*/.*?_(.*)_Packages$";
> +}
> +
> +fn get_changelog_url(
> + package: &str,
> + filename: &str,
> + source_pkg: &str,
> + version: &str,
> + source_version: &str,
> + origin: &str,
> + component: &str,
> +) -> Result<String, Error> {
> + if origin == "" {
> + bail!("no origin available for package {}", package);
> + }
maybe we could just switch this out in favor of 'apt changelog'? or see
what that does internally?
> +
> + if origin == "Debian" {
> + let source_version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(source_version, "");
> +
> + let prefix = if source_pkg.starts_with("lib") {
> + source_pkg.get(0..4)
> + } else {
> + source_pkg.get(0..1)
> + };
> +
> + let prefix = match prefix {
> + Some(p) => p,
> + None => bail!("cannot get starting characters of package name '{}'", package)
> + };
> +
> + // note: security updates seem to not always upload a changelog for
> + // their package version, so this only works *most* of the time
> + return Ok(format!("https://metadata.ftp-master.debian.org/changelogs/main/{}/{}/{}_{}_changelog",
> + prefix, source_pkg, source_pkg, source_version));
> +
> + } else if origin == "Proxmox" && component.starts_with("pbs") {
what about co-located PBS & PVE? and probably less important, what about
the devel/Ceph repositories?
> + let version = (VERSION_EPOCH_REGEX.regex_obj)().replace_all(version, "");
> +
> + let base = match (FILENAME_EXTRACT_REGEX.regex_obj)().captures(filename) {
> + Some(captures) => {
> + let base_capture = captures.get(1);
> + match base_capture {
> + Some(base_underscore) => base_underscore.as_str().replace("_", "/"),
> + None => bail!("incompatible filename, cannot find regex group")
> + }
> + },
> + None => bail!("incompatible filename, doesn't match regex")
> + };
> +
> + return Ok(format!("http://download.proxmox.com/{}/{}_{}.changelog",
> + base, package, version));
> + }
> +
> + bail!("unknown origin ({}) or component ({})", origin, component)
> +}
> +
> +fn list_installed_apt_packages<F: Fn(&str, &str, &str) -> bool>(filter: F)
> + -> Vec<APTUpdateInfo> {
> +
> + let mut ret = Vec::new();
> +
> + // note: this is not an 'apt update', it just re-reads the cache from disk
> + let mut cache = Cache::get_singleton();
> + cache.reload();
> +
> + let mut cache_iter = cache.iter();
> +
> + loop {
> + let view = match cache_iter.next() {
> + Some(view) => view,
> + None => break
> + };
> +
> + let current_version = match view.current_version() {
> + Some(vers) => vers,
> + None => continue
> + };
> + let candidate_version = match view.candidate_version() {
> + Some(vers) => vers,
> + // if there's no candidate (i.e. no update) get info of currently
> + // installed version instead
> + None => current_version.clone()
> + };
> +
> + let package = view.name();
> + if filter(&package, ¤t_version, &candidate_version) {
> + let mut origin_res = "unknown".to_owned();
> + let mut section_res = "unknown".to_owned();
> + let mut priority_res = "unknown".to_owned();
> + let mut change_log_url = "".to_owned();
> + let mut short_desc = package.clone();
> + let mut long_desc = "".to_owned();
> +
> + // get additional information via nested APT 'iterators'
> + let mut view_iter = view.versions();
> + while let Some(ver) = view_iter.next() {
> + if ver.version() == candidate_version {
> + if let Some(section) = ver.section() {
> + section_res = section;
> + }
> +
> + if let Some(prio) = ver.priority_type() {
> + priority_res = prio;
> + }
> +
> + // assume every package has only one origin and package file
isn't this wrong though? e.g., we ship packages that are also shipped
with origin Debian..
> + let mut origin_iter = ver.origin_iter();
> + let origin = origin_iter.next();
> + if let Some(origin) = origin {
> +
> + if let Some(sd) = origin.short_desc() {
> + short_desc = sd;
> + }
> +
> + if let Some(ld) = origin.long_desc() {
> + long_desc = ld;
> + }
> +
> + let mut pkg_iter = origin.file();
> + let pkg_file = pkg_iter.next();
> + if let Some(pkg_file) = pkg_file {
> + if let Some(origin_name) = pkg_file.origin() {
> + origin_res = origin_name;
> + }
> +
> + let filename = pkg_file.file_name();
> + let source_pkg = ver.source_package();
> + let source_ver = ver.source_version();
> + let component = pkg_file.component();
> +
> + // build changelog URL from gathered information
> + // ignore errors, use empty changelog instead
> + let url = get_changelog_url(&package, &filename, &source_pkg,
> + &candidate_version, &source_ver, &origin_res, &component);
> + if let Ok(url) = url {
> + change_log_url = url;
> + }
> + }
> + }
> +
> + break;
> + }
> + }
> +
> + let info = APTUpdateInfo {
> + package,
> + title: short_desc,
> + arch: view.arch(),
> + description: long_desc,
> + change_log_url,
> + origin: origin_res,
> + version: candidate_version,
> + old_version: current_version,
> + priority: priority_res,
> + section: section_res,
> + };
> + ret.push(info);
> + }
> + }
> +
> + return ret;
> +}
> +
> +#[api(
> + input: {
> + properties: {
> + node: {
> + schema: NODE_SCHEMA,
> + },
> + },
> + },
> + returns: {
> + description: "A list of packages with available updates.",
> + type: Array,
> + items: { type: APTUpdateInfo },
> + },
> + access: {
> + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
> + },
> +)]
> +/// List available APT updates
> +fn apt_update_available(_param: Value) -> Result<Value, Error> {
> + let ret = list_installed_apt_packages(|_pkg, cur_ver, can_ver| cur_ver != can_ver);
> + Ok(json!(ret))
> +}
> +
> +const SUBDIRS: SubdirMap = &[
> + ("update", &Router::new().get(&API_METHOD_APT_UPDATE_AVAILABLE)),
> +];
> +
> +pub const ROUTER: Router = Router::new()
> + .get(&list_subdirs_api_method!(SUBDIRS))
> + .subdirs(SUBDIRS);
> diff --git a/src/api2/types.rs b/src/api2/types.rs
> index 0d0fab3b..da73f8db 100644
> --- a/src/api2/types.rs
> +++ b/src/api2/types.rs
> @@ -962,3 +962,30 @@ pub enum RRDTimeFrameResolution {
> /// 1 week => last 490 days
> Year = 60*10080,
> }
> +
> +#[api()]
> +#[derive(Serialize, Deserialize)]
> +#[serde(rename_all = "kebab-case")]
widget toolkit expects PascalCase, so either we fix it there to handle
both (if that's possible), or we return ugly here :-/
> +/// Describes a package for which an update is available.
> +pub struct APTUpdateInfo {
> + /// Package name
> + pub package: String,
> + /// Package title
> + pub title: String,
> + /// Package architecture
> + pub arch: String,
> + /// Human readable package description
> + pub description: String,
> + /// New version to be updated to
> + pub version: String,
> + /// Old version currently installed
> + pub old_version: String,
> + /// Package origin
> + pub origin: String,
> + /// Package priority in human-readable form
> + pub priority: String,
> + /// Package section
> + pub section: String,
> + /// URL under which the package's changelog can be retrieved
> + pub change_log_url: String,
> +}
> --
> 2.20.1
>
>
>
> _______________________________________________
> pbs-devel mailing list
> pbs-devel at lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
>
>
>
More information about the pbs-devel
mailing list