[pbs-devel] [PATCH v3 proxmox-apt 01/10] initial commit
Fabian Ebner
f.ebner at proxmox.com
Mon Mar 22 12:59:36 CET 2021
Signed-off-by: Fabian Ebner <f.ebner at proxmox.com>
---
Changes from v2:
* incorporate Wolfgang's feedback:
* improve warning order/structure in a few places
* make parsing the tail for the one-line format shorter/more readable
* use std::mem::take instead of cloning the comment
* add write_repositories_ref_vec and repositories_from_file helpers for
monomorphization
* have parse_repositories take a &mut Vec and push onto it to avoid
the need to append the result
* improve sorting by matching with std::cmp::Ordering
* improve offset handling for the .sources parser's main loop
* use try_for_each and write directly to avoid collect+join
* implement Display on types instead of From on String
* add comment to explain why AsRef is used
.cargo/config | 5 +
.gitignore | 3 +
Cargo.toml | 22 ++
rustfmt.toml | 1 +
src/lib.rs | 3 +
src/repositories/check.rs | 47 ++++
src/repositories/list_parser.rs | 176 ++++++++++++
src/repositories/mod.rs | 256 ++++++++++++++++++
src/repositories/sources_parser.rs | 214 +++++++++++++++
src/repositories/writer.rs | 85 ++++++
src/types.rs | 211 +++++++++++++++
tests/repositories.rs | 73 +++++
.../sources.list.d.expected/multiline.sources | 8 +
.../options_comment.list | 3 +
.../pbs-enterprise.list | 2 +
tests/sources.list.d.expected/pve.list | 13 +
tests/sources.list.d.expected/standard.list | 7 +
.../sources.list.d.expected/standard.sources | 11 +
tests/sources.list.d/multiline.sources | 9 +
tests/sources.list.d/options_comment.list | 2 +
tests/sources.list.d/pbs-enterprise.list | 1 +
tests/sources.list.d/pve.list | 10 +
tests/sources.list.d/standard.list | 6 +
tests/sources.list.d/standard.sources | 10 +
24 files changed, 1178 insertions(+)
create mode 100644 .cargo/config
create mode 100644 .gitignore
create mode 100644 Cargo.toml
create mode 100644 rustfmt.toml
create mode 100644 src/lib.rs
create mode 100644 src/repositories/check.rs
create mode 100644 src/repositories/list_parser.rs
create mode 100644 src/repositories/mod.rs
create mode 100644 src/repositories/sources_parser.rs
create mode 100644 src/repositories/writer.rs
create mode 100644 src/types.rs
create mode 100644 tests/repositories.rs
create mode 100644 tests/sources.list.d.expected/multiline.sources
create mode 100644 tests/sources.list.d.expected/options_comment.list
create mode 100644 tests/sources.list.d.expected/pbs-enterprise.list
create mode 100644 tests/sources.list.d.expected/pve.list
create mode 100644 tests/sources.list.d.expected/standard.list
create mode 100644 tests/sources.list.d.expected/standard.sources
create mode 100644 tests/sources.list.d/multiline.sources
create mode 100644 tests/sources.list.d/options_comment.list
create mode 100644 tests/sources.list.d/pbs-enterprise.list
create mode 100644 tests/sources.list.d/pve.list
create mode 100644 tests/sources.list.d/standard.list
create mode 100644 tests/sources.list.d/standard.sources
diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 0000000..3b5b6e4
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[source]
+[source.debian-packages]
+directory = "/usr/share/cargo/registry"
+[source.crates-io]
+replace-with = "debian-packages"
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0d00c41
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+Cargo.lock
+target/
+tests/sources.list.d.actual
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..a0ecc26
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "proxmox-apt"
+version = "0.1.0"
+authors = [
+ "Fabian Ebner <f.ebner at proxmox.com>",
+ "Proxmox Support Team <support at proxmox.com>",
+]
+edition = "2018"
+license = "AGPL-3"
+description = "Proxmox library for APT"
+homepage = "https://www.proxmox.com"
+
+exclude = [ "debian" ]
+
+[lib]
+name = "proxmox_apt"
+path = "src/lib.rs"
+
+[dependencies]
+anyhow = "1.0"
+proxmox = { version = "0.11.0", features = [ "api-macro" ] }
+serde = { version = "1.0", features = ["derive"] }
diff --git a/rustfmt.toml b/rustfmt.toml
new file mode 100644
index 0000000..32a9786
--- /dev/null
+++ b/rustfmt.toml
@@ -0,0 +1 @@
+edition = "2018"
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100644
index 0000000..b065c0f
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1,3 @@
+pub mod types;
+
+pub mod repositories;
diff --git a/src/repositories/check.rs b/src/repositories/check.rs
new file mode 100644
index 0000000..87fbbac
--- /dev/null
+++ b/src/repositories/check.rs
@@ -0,0 +1,47 @@
+use anyhow::{bail, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType};
+
+impl APTRepository {
+ /// Makes sure that all basic properties of a repository are present and
+ /// not obviously invalid.
+ pub fn basic_check(&self) -> Result<(), Error> {
+ if self.types.is_empty() {
+ bail!("missing package type(s)");
+ }
+ if self.uris.is_empty() {
+ bail!("missing URI(s)");
+ }
+ if self.suites.is_empty() {
+ bail!("missing suite(s)");
+ }
+
+ for uri in self.uris.iter() {
+ if !uri.contains(':') || uri.len() < 3 {
+ bail!("invalid URI: '{}'", uri);
+ }
+ }
+
+ for suite in self.suites.iter() {
+ if !suite.ends_with('/') && self.components.is_empty() {
+ bail!("missing component(s)");
+ } else if suite.ends_with('/') && !self.components.is_empty() {
+ bail!("absolute suite '{}' does not allow component(s)", suite);
+ }
+ }
+
+ if self.file_type == APTRepositoryFileType::List {
+ if self.types.len() > 1 {
+ bail!("more than one package type");
+ }
+ if self.uris.len() > 1 {
+ bail!("more than one URI");
+ }
+ if self.suites.len() > 1 {
+ bail!("more than one suite");
+ }
+ }
+
+ Ok(())
+ }
+}
diff --git a/src/repositories/list_parser.rs b/src/repositories/list_parser.rs
new file mode 100644
index 0000000..06bb7c2
--- /dev/null
+++ b/src/repositories/list_parser.rs
@@ -0,0 +1,176 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::{Iterator, Peekable};
+use std::str::SplitAsciiWhitespace;
+
+use anyhow::{bail, format_err, Error};
+
+use super::APTRepositoryParser;
+use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryOption};
+
+pub struct APTListFileParser<R: BufRead> {
+ path: String,
+ input: R,
+ line_nr: usize,
+ comment: String,
+}
+
+impl<R: BufRead> APTListFileParser<R> {
+ pub fn new(path: String, reader: R) -> Self {
+ Self {
+ path,
+ input: reader,
+ line_nr: 0,
+ comment: String::new(),
+ }
+ }
+
+ /// Helper to parse options from the existing token stream.
+ /// Also returns `Ok(())` if there are no options.
+ /// Errors when options are invalid or not closed by `']'`.
+ fn parse_options(
+ options: &mut Vec<APTRepositoryOption>,
+ tokens: &mut Peekable<SplitAsciiWhitespace>,
+ ) -> Result<(), Error> {
+ let mut option = match tokens.peek() {
+ Some(token) => {
+ match token.strip_prefix('[') {
+ Some(option) => option,
+ None => return Ok(()), // doesn't look like options
+ }
+ }
+ None => return Ok(()),
+ };
+
+ tokens.next(); // avoid reading the beginning twice
+
+ let mut finished = false;
+ loop {
+ if let Some(stripped) = option.strip_suffix(']') {
+ option = stripped;
+ if option.is_empty() {
+ break;
+ }
+ finished = true; // but still need to handle the last one
+ };
+
+ if let Some(mid) = option.find('=') {
+ let (key, mut value_str) = option.split_at(mid);
+ value_str = &value_str[1..];
+
+ if key.is_empty() {
+ bail!("option has no key: '{}'", option);
+ }
+
+ if value_str.is_empty() {
+ bail!("option has no value: '{}'", option);
+ }
+
+ let values: Vec<String> = value_str
+ .split(',')
+ .map(|value| value.to_string())
+ .collect();
+
+ options.push(APTRepositoryOption {
+ key: key.to_string(),
+ values,
+ });
+ } else if !option.is_empty() {
+ bail!("got invalid option - '{}'", option);
+ }
+
+ if finished {
+ break;
+ }
+
+ option = match tokens.next() {
+ Some(option) => option,
+ None => bail!("options not closed by ']'"),
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Parse a repository or comment in one-line format.
+ /// Commented out repositories are also detected and returned with the
+ /// `enabled` property set to `false`.
+ /// If the line contains a repository, `self.comment` is added to the
+ /// `comment` property.
+ /// If the line contains a comment, it is added to `self.comment`.
+ fn parse_one_line(&mut self, mut line: &str) -> Result<Option<APTRepository>, Error> {
+ line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+ // check for commented out repository first
+ if let Some(commented_out) = line.strip_prefix('#') {
+ if let Ok(Some(mut repo)) = self.parse_one_line(commented_out) {
+ repo.set_enabled(false);
+ return Ok(Some(repo));
+ }
+ }
+
+ let mut repo =
+ APTRepository::new(self.path.clone(), self.line_nr, APTRepositoryFileType::List);
+
+ // now handle "real" comment
+ if let Some(comment_start) = line.find('#') {
+ let (line_start, comment) = line.split_at(comment_start);
+ self.comment = format!("{}{}\n", self.comment, &comment[1..]);
+ line = line_start;
+ }
+
+ let mut tokens = line.split_ascii_whitespace().peekable();
+
+ match tokens.next() {
+ Some(package_type) => {
+ repo.types.push(package_type.try_into()?);
+ }
+ None => return Ok(None), // empty line
+ }
+
+ Self::parse_options(&mut repo.options, &mut tokens)?;
+
+ // the rest of the line is just '<uri> <suite> [<components>...]'
+ let mut tokens = tokens.map(str::to_string);
+ repo.uris
+ .push(tokens.next().ok_or_else(|| format_err!("missing URI"))?);
+ repo.suites
+ .push(tokens.next().ok_or_else(|| format_err!("missing suite"))?);
+ repo.components.extend(tokens);
+
+ repo.comment = std::mem::take(&mut self.comment);
+
+ Ok(Some(repo))
+ }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTListFileParser<R> {
+ fn parse_repositories(&mut self, repos: &mut Vec<APTRepository>) -> Result<(), Error> {
+ let mut line = String::new();
+
+ loop {
+ self.line_nr += 1;
+ line.clear();
+
+ match self.input.read_line(&mut line) {
+ Err(err) => bail!("input error for '{}' - {}", self.path, err),
+ Ok(0) => break,
+ Ok(_) => match self.parse_one_line(&line) {
+ Ok(Some(repo)) => {
+ repos.push(repo);
+ self.comment.clear();
+ }
+ Ok(None) => continue,
+ Err(err) => bail!(
+ "malformed entry in '{}' line {} - {}",
+ self.path,
+ self.line_nr,
+ err,
+ ),
+ },
+ }
+ }
+
+ Ok(())
+ }
+}
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
new file mode 100644
index 0000000..5c446af
--- /dev/null
+++ b/src/repositories/mod.rs
@@ -0,0 +1,256 @@
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+use std::convert::TryFrom;
+use std::ffi::OsString;
+use std::path::{Path, PathBuf};
+
+use anyhow::{bail, format_err, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryOption};
+
+mod list_parser;
+use list_parser::APTListFileParser;
+
+mod sources_parser;
+use sources_parser::APTSourcesFileParser;
+
+mod check;
+mod writer;
+
+const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list";
+const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/";
+
+impl APTRepository {
+ /// Crates an empty repository with infomration that is known before parsing.
+ fn new(path: String, number: usize, file_type: APTRepositoryFileType) -> Self {
+ Self {
+ types: vec![],
+ uris: vec![],
+ suites: vec![],
+ components: vec![],
+ options: vec![],
+ comment: String::new(),
+ path,
+ number,
+ file_type,
+ enabled: true,
+ }
+ }
+
+ /// Changes the `enabled` flag and makes sure the `Enabled` option for
+ /// `APTRepositoryPackageType::Sources` repositories is updated too.
+ fn set_enabled(&mut self, enabled: bool) {
+ self.enabled = enabled;
+
+ if self.file_type == APTRepositoryFileType::Sources {
+ let enabled_string = match enabled {
+ true => "true".to_string(),
+ false => "false".to_string(),
+ };
+ for option in self.options.iter_mut() {
+ if option.key == "Enabled" {
+ option.values = vec![enabled_string];
+ return;
+ }
+ }
+ self.options.push(APTRepositoryOption {
+ key: "Enabled".to_string(),
+ values: vec![enabled_string],
+ });
+ }
+ }
+}
+
+trait APTRepositoryParser {
+ /// Parse all repositories including the disabled ones and push them onto
+ /// the provided vector.
+ fn parse_repositories(&mut self, repos: &mut Vec<APTRepository>) -> Result<(), Error>;
+}
+
+/// Helper to decide whether a file name is considered valid by APT and to
+/// extract its file type and the path as a string.
+/// Hidden files yield `Ok(None)`, while invalid file names yield an error.
+fn check_filename<P: AsRef<Path>>(
+ path: P,
+) -> Result<Option<(APTRepositoryFileType, String)>, OsString> {
+ let path: PathBuf = path.as_ref().to_path_buf();
+ let path_string = path.clone().into_os_string().into_string()?;
+
+ let file_name = match path.file_name() {
+ Some(file_name) => file_name.to_os_string().into_string()?,
+ None => return Err(OsString::from(path_string)),
+ };
+
+ // APT silently ignores hidden files
+ if file_name.starts_with('.') {
+ return Ok(None);
+ }
+
+ let extension = match path.extension() {
+ Some(extension) => extension.to_os_string().into_string()?,
+ None => return Err(OsString::from(path_string)),
+ };
+
+ let file_type = match APTRepositoryFileType::try_from(&extension[..]) {
+ Ok(file_type) => file_type,
+ _ => return Err(OsString::from(path_string)),
+ };
+
+ // APT ignores such files but issues a warning
+ if !file_name
+ .chars()
+ .all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '-' || x == '.')
+ {
+ return Err(OsString::from(path_string));
+ }
+
+ Ok(Some((file_type, path_string)))
+}
+
+/// Similar to [`repositories_from_files`], but for a single file, and adds the
+/// parsed repositories onto the provided vector instead. Another difference is
+/// that it doesn't call [`basic_check`](check::basic_check).
+fn repositories_from_file(path: &Path, repos: &mut Vec<APTRepository>) -> Result<(), Error> {
+ if !path.is_file() {
+ eprintln!("Ignoring {:?} - not a file", path);
+ return Ok(());
+ }
+
+ let file_type;
+ let path_string;
+
+ match check_filename(path) {
+ Ok(Some(res)) => {
+ file_type = res.0;
+ path_string = res.1;
+ }
+ Ok(None) => return Ok(()),
+ Err(path) => {
+ eprintln!("Ignoring {:?} - invalid file name", path);
+ return Ok(());
+ }
+ }
+
+ let contents =
+ std::fs::read(path).map_err(|err| format_err!("unable to read {:?} - {}", path, err))?;
+
+ let mut parser: Box<dyn APTRepositoryParser> = match file_type {
+ APTRepositoryFileType::List => Box::new(APTListFileParser::new(path_string, &contents[..])),
+ APTRepositoryFileType::Sources => {
+ Box::new(APTSourcesFileParser::new(path_string, &contents[..]))
+ }
+ };
+
+ parser.parse_repositories(repos)?;
+
+ Ok(())
+}
+
+/// Returns all APT repositories configured in the specified files, including
+/// disabled ones.
+/// Warns about invalid file names and some format violations, while other
+/// format violations result in an error.
+pub fn repositories_from_files<P: AsRef<Path>>(paths: &[P]) -> Result<Vec<APTRepository>, Error> {
+ let mut repos = Vec::<APTRepository>::new();
+
+ for path in paths.iter() {
+ repositories_from_file(path.as_ref(), &mut repos)?;
+ }
+
+ for repo in repos.iter() {
+ repo.basic_check().map_err(|err| {
+ format_err!("check for {}:{} failed - {}", repo.path, repo.number, err)
+ })?;
+ }
+
+ Ok(repos)
+}
+
+/// Returns all APT repositories configured in `/etc/apt/sources.list` and
+/// in `/etc/apt/sources.list.d` including disabled repositories.
+/// Warns about invalid file names and some format violations, while other
+/// format violations result in an error.
+pub fn repositories() -> Result<Vec<APTRepository>, Error> {
+ let mut paths = Vec::<PathBuf>::new();
+
+ let sources_list_path = PathBuf::from(APT_SOURCES_LIST_FILENAME);
+ if sources_list_path.is_file() {
+ paths.push(sources_list_path)
+ };
+
+ let sources_list_d_path = PathBuf::from(APT_SOURCES_LIST_DIRECTORY);
+ if sources_list_d_path.is_dir() {
+ for entry in std::fs::read_dir(sources_list_d_path)? {
+ paths.push(entry?.path());
+ }
+ }
+
+ let repos = repositories_from_files(&paths)?;
+
+ Ok(repos)
+}
+
+/// See [`write_repositories`]. Will sort the vector by the repository's file
+/// name and number.
+fn write_repositories_ref_vec(repos: &mut Vec<&APTRepository>) -> Result<(), Error> {
+ // check before writing
+ for repo in repos.iter() {
+ repo.basic_check().map_err(|err| {
+ format_err!("check for {}:{} failed - {}", repo.path, repo.number, err)
+ })?;
+ }
+
+ repos.sort_by(|a, b| match a.path.cmp(&b.path) {
+ Ordering::Equal => a.number.cmp(&b.number),
+ ord => ord,
+ });
+
+ let mut files = BTreeMap::<String, Vec<u8>>::new();
+
+ for repo in repos.iter() {
+ let raw = match files.get_mut(&repo.path) {
+ Some(raw) => raw,
+ None => {
+ files.insert(repo.path.clone(), vec![]);
+ files.get_mut(&repo.path).unwrap()
+ }
+ };
+
+ repo.write(&mut *raw)
+ .map_err(|err| format_err!("writing {}:{} failed - {}", repo.path, repo.number, err))?;
+ }
+
+ for (path, content) in files.iter() {
+ let path = PathBuf::from(path);
+ let dir = path.parent().unwrap();
+
+ std::fs::create_dir_all(dir)
+ .map_err(|err| format_err!("unable to create dir {:?} - {}", dir, err))?;
+
+ let pid = std::process::id();
+ let mut tmp_path = path.clone();
+ tmp_path.set_extension("tmp");
+ tmp_path.set_extension(format!("{}", pid));
+
+ if let Err(err) = std::fs::write(&tmp_path, &content) {
+ let _ = std::fs::remove_file(&tmp_path);
+ bail!("write failed: {}", err);
+ }
+
+ if let Err(err) = std::fs::rename(&tmp_path, &path) {
+ let _ = std::fs::remove_file(&tmp_path);
+ bail!("rename failed for {:?} - {}", path, err);
+ }
+ }
+
+ Ok(())
+}
+
+/// Write the repositories to the respective files specified by their
+/// `path` property and in the order determined by their `number` property.
+/// Does a `check::basic_check(repository)` for each repository first.
+pub fn write_repositories<A: AsRef<APTRepository>>(repos: &[A]) -> Result<(), Error> {
+ let mut repos: Vec<&APTRepository> = repos.iter().map(|repo| repo.as_ref()).collect();
+
+ write_repositories_ref_vec(&mut repos)
+}
diff --git a/src/repositories/sources_parser.rs b/src/repositories/sources_parser.rs
new file mode 100644
index 0000000..5f25d33
--- /dev/null
+++ b/src/repositories/sources_parser.rs
@@ -0,0 +1,214 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::Iterator;
+
+use anyhow::{bail, Error};
+
+use crate::types::{
+ APTRepository, APTRepositoryFileType, APTRepositoryOption, APTRepositoryPackageType,
+};
+
+use super::APTRepositoryParser;
+
+pub struct APTSourcesFileParser<R: BufRead> {
+ path: String,
+ input: R,
+ stanza_nr: usize,
+ comment: String,
+}
+
+/// See `man sources.list` and `man deb822` for the format specification.
+impl<R: BufRead> APTSourcesFileParser<R> {
+ pub fn new(path: String, reader: R) -> Self {
+ Self {
+ path,
+ input: reader,
+ stanza_nr: 1,
+ comment: String::new(),
+ }
+ }
+
+ /// Based on APT's `StringToBool` in `strutl.cc`
+ fn string_to_bool(string: &str, default: bool) -> bool {
+ let string = string.trim_matches(|c| char::is_ascii_whitespace(&c));
+ let string = string.to_lowercase();
+
+ match &string[..] {
+ "1" | "yes" | "true" | "with" | "on" | "enable" => true,
+ "0" | "no" | "false" | "without" | "off" | "disable" => false,
+ _ => default,
+ }
+ }
+
+ /// Checks if `key` is valid according to deb822
+ fn valid_key(key: &str) -> bool {
+ if key.starts_with('-') {
+ return false;
+ };
+ return key.chars().all(|c| matches!(c, '!'..='9' | ';'..='~'));
+ }
+
+ /// Try parsing a repository in stanza format from `lines`.
+ /// Returns `Ok(None)` when no stanza can be found.
+ /// Comments are added to `self.comments`. If a stanza can be found,
+ /// `self.comment` is added to the repository's `comment` property.
+ /// Fully commented out stanzas are treated as comments.
+ fn parse_stanza(&mut self, lines: &str) -> Result<Option<APTRepository>, Error> {
+ let mut repo = APTRepository::new(
+ self.path.clone(),
+ self.stanza_nr,
+ APTRepositoryFileType::Sources,
+ );
+
+ // Values may be folded into multiple lines.
+ // Those lines have to start with a space or a tab.
+ let lines = lines.replace("\n ", " ");
+ let lines = lines.replace("\n\t", " ");
+
+ let mut got_something = false;
+
+ for line in lines.lines() {
+ let line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+ if line.is_empty() {
+ continue;
+ }
+
+ if let Some(commented_out) = line.strip_prefix('#') {
+ self.comment = format!("{}{}\n", self.comment, commented_out);
+ continue;
+ }
+
+ if let Some(mid) = line.find(':') {
+ let (key, value_str) = line.split_at(mid);
+ let value_str = &value_str[1..];
+ let key = key.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+ if key.is_empty() {
+ bail!("option has no key: '{}'", line);
+ }
+
+ if value_str.is_empty() {
+ // ignored by APT
+ eprintln!("option has no value: '{}'", line);
+ continue;
+ }
+
+ if !Self::valid_key(key) {
+ // ignored by APT
+ eprintln!("option with invalid key '{}'", key);
+ continue;
+ }
+
+ let values: Vec<String> = value_str
+ .split_ascii_whitespace()
+ .map(|value| value.to_string())
+ .collect();
+
+ match key {
+ "Types" => {
+ if !repo.types.is_empty() {
+ eprintln!("key 'Types' was defined twice");
+ }
+ let mut types = Vec::<APTRepositoryPackageType>::new();
+ for package_type in values {
+ types.push((&package_type[..]).try_into()?);
+ }
+ repo.types = types;
+ }
+ "URIs" => {
+ if !repo.uris.is_empty() {
+ eprintln!("key 'URIs' was defined twice");
+ }
+ repo.uris = values;
+ }
+ "Suites" => {
+ if !repo.suites.is_empty() {
+ eprintln!("key 'Suites' was defined twice");
+ }
+ repo.suites = values;
+ }
+ "Components" => {
+ if !repo.components.is_empty() {
+ eprintln!("key 'Components' was defined twice");
+ }
+ repo.components = values;
+ }
+ "Enabled" => {
+ repo.set_enabled(Self::string_to_bool(value_str, true));
+ }
+ _ => repo.options.push(APTRepositoryOption {
+ key: key.to_string(),
+ values,
+ }),
+ }
+ } else {
+ bail!("got invalid line - '{:?}'", line);
+ }
+
+ got_something = true;
+ }
+
+ if !got_something {
+ return Ok(None);
+ }
+
+ repo.comment = std::mem::take(&mut self.comment);
+
+ Ok(Some(repo))
+ }
+
+ /// Helper function for `parse_repositories`.
+ fn try_parse_stanza(
+ &mut self,
+ lines: &str,
+ repos: &mut Vec<APTRepository>,
+ ) -> Result<(), Error> {
+ match self.parse_stanza(&lines[..]) {
+ Ok(Some(repo)) => {
+ repos.push(repo);
+ self.comment.clear();
+ self.stanza_nr += 1;
+ }
+ Ok(None) => (),
+ Err(err) => {
+ bail!(
+ "malformed entry in '{}' stanza {} - {}",
+ self.path,
+ self.stanza_nr,
+ err,
+ );
+ }
+ }
+
+ Ok(())
+ }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTSourcesFileParser<R> {
+ fn parse_repositories(&mut self, repos: &mut Vec<APTRepository>) -> Result<(), Error> {
+ let mut lines = String::new();
+
+ loop {
+ let old_length = lines.len();
+ match self.input.read_line(&mut lines) {
+ Err(err) => bail!("input error for '{}' - {}", self.path, err),
+ Ok(0) => {
+ self.try_parse_stanza(&lines[..], repos)?;
+ break;
+ }
+ Ok(_) => {
+ if (&lines[old_length..])
+ .trim_matches(|c| char::is_ascii_whitespace(&c))
+ .is_empty()
+ {
+ // detected end of stanza
+ self.try_parse_stanza(&lines[..], repos)?;
+ lines.clear();
+ }
+ }
+ }
+ }
+
+ Ok(())
+ }
+}
diff --git a/src/repositories/writer.rs b/src/repositories/writer.rs
new file mode 100644
index 0000000..76ea6ea
--- /dev/null
+++ b/src/repositories/writer.rs
@@ -0,0 +1,85 @@
+use std::io::Write;
+
+use anyhow::{bail, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType};
+
+impl APTRepository {
+ /// Writes a repository in the corresponding format followed by a blank.
+ /// Expects that `basic_check()` for the repository was successful.
+ pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> {
+ match self.file_type {
+ APTRepositoryFileType::List => write_one_line(self, w),
+ APTRepositoryFileType::Sources => write_stanza(self, w),
+ }
+ }
+}
+
+/// Writes a repository in one-line format followed by a blank line.
+/// Expects that `repo.file_type == APTRepositoryFileType::List`.
+/// Expects that `basic_check()` for the repository was successful.
+fn write_one_line(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+ if repo.file_type != APTRepositoryFileType::List {
+ bail!("not a .list repository");
+ }
+
+ if !repo.comment.is_empty() {
+ for line in repo.comment.lines() {
+ writeln!(w, "#{}", line)?;
+ }
+ }
+
+ if !repo.enabled {
+ write!(w, "# ")?;
+ }
+
+ write!(w, "{} ", repo.types[0])?;
+
+ if !repo.options.is_empty() {
+ write!(w, "[ ")?;
+ repo.options
+ .iter()
+ .try_for_each(|option| write!(w, "{}={} ", option.key, option.values.join(",")))?;
+ write!(w, "] ")?;
+ };
+
+ write!(w, "{} ", repo.uris[0])?;
+ write!(w, "{} ", repo.suites[0])?;
+ writeln!(w, "{}", repo.components.join(" "))?;
+
+ writeln!(w)?;
+
+ Ok(())
+}
+
+/// Writes a single stanza followed by a blank line.
+/// Expects that `repo.file_type == APTRepositoryFileType::Sources`.
+fn write_stanza(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+ if repo.file_type != APTRepositoryFileType::Sources {
+ bail!("not a .sources repository");
+ }
+
+ if !repo.comment.is_empty() {
+ for line in repo.comment.lines() {
+ writeln!(w, "#{}", line)?;
+ }
+ }
+
+ write!(w, "Types:")?;
+ repo.types
+ .iter()
+ .try_for_each(|package_type| write!(w, " {}", package_type))?;
+ writeln!(w)?;
+
+ writeln!(w, "URIs: {}", repo.uris.join(" "))?;
+ writeln!(w, "Suites: {}", repo.suites.join(" "))?;
+ writeln!(w, "Components: {}", repo.components.join(" "))?;
+
+ for option in repo.options.iter() {
+ writeln!(w, "{}: {}", option.key, option.values.join(" "))?;
+ }
+
+ writeln!(w)?;
+
+ Ok(())
+}
diff --git a/src/types.rs b/src/types.rs
new file mode 100644
index 0000000..be69652
--- /dev/null
+++ b/src/types.rs
@@ -0,0 +1,211 @@
+use std::convert::TryFrom;
+use std::fmt::Display;
+
+use anyhow::{bail, Error};
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum APTRepositoryFileType {
+ /// One-line-style format
+ List,
+ /// DEB822-style format
+ Sources,
+}
+
+impl TryFrom<&str> for APTRepositoryFileType {
+ type Error = Error;
+
+ fn try_from(string: &str) -> Result<Self, Error> {
+ match &string[..] {
+ "list" => Ok(APTRepositoryFileType::List),
+ "sources" => Ok(APTRepositoryFileType::Sources),
+ _ => bail!("invalid file type '{}'", string),
+ }
+ }
+}
+
+impl Display for APTRepositoryFileType {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ APTRepositoryFileType::List => write!(f, "list"),
+ APTRepositoryFileType::Sources => write!(f, "sources"),
+ }
+ }
+}
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+pub enum APTRepositoryPackageType {
+ /// Debian package
+ Deb,
+ /// Debian source package
+ DebSrc,
+}
+
+impl TryFrom<&str> for APTRepositoryPackageType {
+ type Error = Error;
+
+ fn try_from(string: &str) -> Result<Self, Error> {
+ match &string[..] {
+ "deb" => Ok(APTRepositoryPackageType::Deb),
+ "deb-src" => Ok(APTRepositoryPackageType::DebSrc),
+ _ => bail!("invalid file type '{}'", string),
+ }
+ }
+}
+
+impl Display for APTRepositoryPackageType {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ APTRepositoryPackageType::Deb => write!(f, "deb"),
+ APTRepositoryPackageType::DebSrc => write!(f, "deb-src"),
+ }
+ }
+}
+
+#[api(
+ properties: {
+ Key: {
+ description: "Option key.",
+ type: String,
+ },
+ Values: {
+ description: "Option values.",
+ type: Array,
+ items: {
+ description: "Value.",
+ type: String,
+ },
+ },
+ },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")] // for consistency
+/// Additional options for an APT repository.
+/// Used for both single- and mutli-value options.
+pub struct APTRepositoryOption {
+ /// Option key.
+ pub key: String,
+ /// Option value(s).
+ pub values: Vec<String>,
+}
+
+#[api(
+ properties: {
+ Types: {
+ description: "List of package types.",
+ type: Array,
+ items: {
+ type: APTRepositoryPackageType,
+ },
+ },
+ URIs: {
+ description: "List of repository URIs.",
+ type: Array,
+ items: {
+ description: "Repository URI.",
+ type: String,
+ },
+ },
+ Suites: {
+ description: "List of distributions.",
+ type: Array,
+ items: {
+ description: "Package distribution.",
+ type: String,
+ },
+ },
+ Components: {
+ description: "List of repository components.",
+ type: Array,
+ items: {
+ description: "Repository component.",
+ type: String,
+ },
+ },
+ Options: {
+ type: Array,
+ optional: true,
+ items: {
+ type: APTRepositoryOption,
+ },
+ },
+ Comment: {
+ description: "Associated comment.",
+ type: String,
+ optional: true,
+ },
+ Path: {
+ description: "Path to the defining file.",
+ type: String,
+ },
+ Number: {
+ description: "Line or stanza number.",
+ type: Integer,
+ },
+ FileType: {
+ type: APTRepositoryFileType,
+ },
+ Enabled: {
+ description: "Whether the repository is enabled or not.",
+ type: Boolean,
+ },
+ },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+/// Describes an APT repository.
+pub struct APTRepository {
+ /// List of package types.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub types: Vec<APTRepositoryPackageType>,
+
+ /// List of repository URIs.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ #[serde(rename = "URIs")]
+ pub uris: Vec<String>,
+
+ /// List of package distributions.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub suites: Vec<String>,
+
+ /// List of repository components.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub components: Vec<String>,
+
+ /// Additional options.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub options: Vec<APTRepositoryOption>,
+
+ /// Associated comment.
+ #[serde(skip_serializing_if = "String::is_empty")]
+ pub comment: String,
+
+ /// Path to the defining file.
+ #[serde(skip_serializing_if = "String::is_empty")]
+ pub path: String,
+
+ /// Line or stanza number.
+ pub number: usize,
+
+ /// Format of the defining file.
+ pub file_type: APTRepositoryFileType,
+
+ /// Whether the repository is enabled or not.
+ pub enabled: bool,
+}
+
+/// Some functions like write_repositiories can be called with either a slice of
+/// [`APTRepository`]s or a slice of references thereof. Thus, users of the
+/// crate are more flexibility in working with collections of repositories. See
+/// the test_parse_write test for an example.
+impl AsRef<APTRepository> for APTRepository {
+ fn as_ref(&self) -> &APTRepository {
+ &self
+ }
+}
diff --git a/tests/repositories.rs b/tests/repositories.rs
new file mode 100644
index 0000000..020e133
--- /dev/null
+++ b/tests/repositories.rs
@@ -0,0 +1,73 @@
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox_apt::repositories::{repositories_from_files, write_repositories};
+use proxmox_apt::types::APTRepository;
+
+#[test]
+fn test_parse_write() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+ let write_dir = test_dir.join("sources.list.d.actual");
+ let expected_dir = test_dir.join("sources.list.d.expected");
+
+ let mut paths = Vec::<PathBuf>::new();
+ for entry in std::fs::read_dir(read_dir)? {
+ paths.push(entry?.path());
+ }
+
+ let repos = repositories_from_files(&paths)?;
+
+ // used to mess up the order from parsing and to check that each repo has a
+ // unique path:number
+ let mut repo_hash = HashMap::<String, APTRepository>::new();
+
+ for mut repo in repos {
+ let path = PathBuf::from(repo.path);
+ let new_path = write_dir.join(path.file_name().unwrap());
+
+ repo.path = new_path.into_os_string().into_string().unwrap();
+
+ let key = format!("{}:{}", repo.path, repo.number);
+
+ if repo_hash.insert(key.clone(), repo).is_some() {
+ bail!("key '{}' is not unique!", key);
+ }
+ }
+
+ if write_dir.is_dir() {
+ std::fs::remove_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+ }
+
+ std::fs::create_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+ let repos_vec: Vec<&APTRepository> = repo_hash.values().collect();
+ write_repositories(&repos_vec)?;
+
+ let mut expected_count = 0;
+
+ for entry in std::fs::read_dir(expected_dir)? {
+ expected_count += 1;
+
+ let expected_path = entry?.path();
+ let actual_path = write_dir.join(expected_path.file_name().unwrap());
+
+ let expected_contents = std::fs::read(&expected_path)
+ .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?;
+
+ let actual_contents = std::fs::read(&actual_path)
+ .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?;
+
+ assert_eq!(expected_contents, actual_contents);
+ }
+
+ let actual_count = std::fs::read_dir(write_dir)?.count();
+
+ assert_eq!(expected_count, actual_count);
+
+ Ok(())
+}
diff --git a/tests/sources.list.d.expected/multiline.sources b/tests/sources.list.d.expected/multiline.sources
new file mode 100644
index 0000000..91f53c2
--- /dev/null
+++ b/tests/sources.list.d.expected/multiline.sources
@@ -0,0 +1,8 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+Languages: it de fr
+Enabled: false
+
diff --git a/tests/sources.list.d.expected/options_comment.list b/tests/sources.list.d.expected/options_comment.list
new file mode 100644
index 0000000..f0952e4
--- /dev/null
+++ b/tests/sources.list.d.expected/options_comment.list
@@ -0,0 +1,3 @@
+# comment
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian buster main contrib
+
diff --git a/tests/sources.list.d.expected/pbs-enterprise.list b/tests/sources.list.d.expected/pbs-enterprise.list
new file mode 100644
index 0000000..acb2990
--- /dev/null
+++ b/tests/sources.list.d.expected/pbs-enterprise.list
@@ -0,0 +1,2 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
+
diff --git a/tests/sources.list.d.expected/pve.list b/tests/sources.list.d.expected/pve.list
new file mode 100644
index 0000000..127a49a
--- /dev/null
+++ b/tests/sources.list.d.expected/pve.list
@@ -0,0 +1,13 @@
+deb http://ftp.debian.org/debian buster main contrib
+
+deb http://ftp.debian.org/debian buster-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security buster/updates main contrib
+
diff --git a/tests/sources.list.d.expected/standard.list b/tests/sources.list.d.expected/standard.list
new file mode 100644
index 0000000..63c1b60
--- /dev/null
+++ b/tests/sources.list.d.expected/standard.list
@@ -0,0 +1,7 @@
+deb http://ftp.at.debian.org/debian buster main contrib
+
+deb http://ftp.at.debian.org/debian buster-updates main contrib
+
+# security updates
+deb http://security.debian.org buster/updates main contrib
+
diff --git a/tests/sources.list.d.expected/standard.sources b/tests/sources.list.d.expected/standard.sources
new file mode 100644
index 0000000..56ce280
--- /dev/null
+++ b/tests/sources.list.d.expected/standard.sources
@@ -0,0 +1,11 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: buster/updates
+Components: main contrib
+
diff --git a/tests/sources.list.d/multiline.sources b/tests/sources.list.d/multiline.sources
new file mode 100644
index 0000000..c3a1ff0
--- /dev/null
+++ b/tests/sources.list.d/multiline.sources
@@ -0,0 +1,9 @@
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+# comment in here
+Components: main contrib
+Languages: it
+ de
+ fr
+Enabled: off
diff --git a/tests/sources.list.d/options_comment.list b/tests/sources.list.d/options_comment.list
new file mode 100644
index 0000000..e3f4112
--- /dev/null
+++ b/tests/sources.list.d/options_comment.list
@@ -0,0 +1,2 @@
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian buster main contrib # comment
+
diff --git a/tests/sources.list.d/pbs-enterprise.list b/tests/sources.list.d/pbs-enterprise.list
new file mode 100644
index 0000000..5f8763c
--- /dev/null
+++ b/tests/sources.list.d/pbs-enterprise.list
@@ -0,0 +1 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
diff --git a/tests/sources.list.d/pve.list b/tests/sources.list.d/pve.list
new file mode 100644
index 0000000..6213f72
--- /dev/null
+++ b/tests/sources.list.d/pve.list
@@ -0,0 +1,10 @@
+deb http://ftp.debian.org/debian buster main contrib
+deb http://ftp.debian.org/debian buster-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security buster/updates main contrib
diff --git a/tests/sources.list.d/standard.list b/tests/sources.list.d/standard.list
new file mode 100644
index 0000000..26db887
--- /dev/null
+++ b/tests/sources.list.d/standard.list
@@ -0,0 +1,6 @@
+deb http://ftp.at.debian.org/debian buster main contrib
+
+deb http://ftp.at.debian.org/debian buster-updates main contrib
+
+# security updates
+deb http://security.debian.org buster/updates main contrib
diff --git a/tests/sources.list.d/standard.sources b/tests/sources.list.d/standard.sources
new file mode 100644
index 0000000..605202e
--- /dev/null
+++ b/tests/sources.list.d/standard.sources
@@ -0,0 +1,10 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: buster/updates
+Components: main contrib
--
2.20.1
More information about the pbs-devel
mailing list