[pve-devel] [PATCH proxmox-apt 2/2] deb822: source index support
Fabian Grünbichler
f.gruenbichler at proxmox.com
Tue Oct 18 11:20:36 CEST 2022
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
the test file needs to be downloaded from the referenced URL and
uncompressed (it's too big to send as patch).
its SHA256sum is e7777c1d305f5e0a31bcf2fe26e955436986edb5c211c03a362c7d557c899349
src/deb822/mod.rs | 3 +
src/deb822/release_file.rs | 2 +-
src/deb822/sources_file.rs | 255 +
..._debian_dists_bullseye_main_source_Sources | 858657 +++++++++++++++
4 files changed, 858916 insertions(+), 1 deletion(-)
create mode 100644 src/deb822/sources_file.rs
create mode 100644 tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources
diff --git a/src/deb822/mod.rs b/src/deb822/mod.rs
index 7a1bb0e..59e7c21 100644
--- a/src/deb822/mod.rs
+++ b/src/deb822/mod.rs
@@ -5,6 +5,9 @@ pub use release_file::{CompressionType, FileReference, FileReferenceType, Releas
mod packages_file;
pub use packages_file::PackagesFile;
+mod sources_file;
+pub use sources_file::SourcesFile;
+
#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
pub struct CheckSums {
pub md5: Option<[u8; 16]>,
diff --git a/src/deb822/release_file.rs b/src/deb822/release_file.rs
index c50c095..85d3436 100644
--- a/src/deb822/release_file.rs
+++ b/src/deb822/release_file.rs
@@ -245,7 +245,7 @@ impl FileReferenceType {
}
pub fn is_package_index(&self) -> bool {
- matches!(self, FileReferenceType::Packages(_, _))
+ matches!(self, FileReferenceType::Packages(_, _) | FileReferenceType::Sources(_))
}
}
diff --git a/src/deb822/sources_file.rs b/src/deb822/sources_file.rs
new file mode 100644
index 0000000..a13d84f
--- /dev/null
+++ b/src/deb822/sources_file.rs
@@ -0,0 +1,255 @@
+use std::collections::HashMap;
+
+use anyhow::{bail, Error, format_err};
+use rfc822_like::de::Deserializer;
+use serde::Deserialize;
+use serde_json::Value;
+
+use super::CheckSums;
+//Uploaders
+//
+//Homepage
+//
+//Version Control System (VCS) fields
+//
+//Testsuite
+//
+//Dgit
+//
+//Standards-Version (mandatory)
+//
+//Build-Depends et al
+//
+//Package-List (recommended)
+//
+//Checksums-Sha1 and Checksums-Sha256 (mandatory)
+//
+//Files (mandatory)
+
+
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct SourcesFileRaw {
+ pub format: String,
+ pub package: String,
+ pub binary: Option<Vec<String>>,
+ pub version: String,
+ pub section: Option<String>,
+ pub priority: Option<String>,
+ pub maintainer: String,
+ pub uploaders: Option<String>,
+ pub architecture: Option<String>,
+ pub directory: String,
+ pub files: String,
+ #[serde(rename = "Checksums-Sha256")]
+ pub sha256: Option<String>,
+ #[serde(rename = "Checksums-Sha512")]
+ pub sha512: Option<String>,
+ #[serde(flatten)]
+ pub extra_fields: HashMap<String, Value>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct SourcePackageEntry {
+ pub format: String,
+ pub package: String,
+ pub binary: Option<Vec<String>>,
+ pub version: String,
+ pub architecture: Option<String>,
+ pub section: Option<String>,
+ pub priority: Option<String>,
+ pub maintainer: String,
+ pub uploaders: Option<String>,
+ pub directory: String,
+ pub files: HashMap<String, SourcePackageFileReference>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct SourcePackageFileReference {
+ pub file: String,
+ pub size: usize,
+ pub checksums: CheckSums,
+}
+
+impl SourcePackageEntry {
+ pub fn size(&self) -> usize {
+ self.files.values().map(|f| f.size).sum()
+ }
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+/// A parsed representation of a Release file
+pub struct SourcesFile {
+ pub source_packages: Vec<SourcePackageEntry>,
+}
+
+impl TryFrom<SourcesFileRaw> for SourcePackageEntry {
+ type Error = Error;
+
+ fn try_from(value: SourcesFileRaw) -> Result<Self, Self::Error> {
+ let mut parsed = SourcePackageEntry {
+ package: value.package,
+ binary: value.binary,
+ version: value.version,
+ architecture: value.architecture,
+ files: HashMap::new(),
+ format: value.format,
+ section: value.section,
+ priority: value.priority,
+ maintainer: value.maintainer,
+ uploaders: value.uploaders,
+ directory: value.directory,
+ };
+
+ for file_reference in value.files.lines() {
+ let (file_name, size, md5) = parse_file_reference(file_reference, 16)?;
+ let entry = parsed.files.entry(file_name.clone()).or_insert_with(|| SourcePackageFileReference { file: file_name, size, checksums: CheckSums::default()});
+ entry.checksums.md5 = Some(md5.try_into().map_err(|_|format_err!("unexpected checksum length"))?);
+ if entry.size != size {
+ bail!("Size mismatch: {} != {}", entry.size, size);
+ }
+ }
+
+ if let Some(sha256) = value.sha256 {
+ for line in sha256.lines() {
+ let (file_name, size, sha256) = parse_file_reference(line, 32)?;
+ let entry = parsed.files.entry(file_name.clone()).or_insert_with(|| SourcePackageFileReference { file: file_name, size, checksums: CheckSums::default()});
+ entry.checksums.sha256 = Some(sha256.try_into().map_err(|_|format_err!("unexpected checksum length"))?);
+ if entry.size != size {
+ bail!("Size mismatch: {} != {}", entry.size, size);
+ }
+ }
+ };
+
+ if let Some(sha512) = value.sha512 {
+ for line in sha512.lines() {
+ let (file_name, size, sha512) = parse_file_reference(line, 64)?;
+ let entry = parsed.files.entry(file_name.clone()).or_insert_with(|| SourcePackageFileReference { file: file_name, size, checksums: CheckSums::default()});
+ entry.checksums.sha512 = Some(sha512.try_into().map_err(|_|format_err!("unexpected checksum length"))?);
+ if entry.size != size {
+ bail!("Size mismatch: {} != {}", entry.size, size);
+ }
+ }
+ };
+
+ for (file_name, reference) in &parsed.files {
+ if !reference.checksums.is_secure() {
+ bail!(
+ "no strong checksum found for source entry '{}'",
+ file_name
+ );
+ }
+ }
+
+ Ok(parsed)
+ }
+}
+
+impl TryFrom<String> for SourcesFile {
+ type Error = Error;
+
+ fn try_from(value: String) -> Result<Self, Self::Error> {
+ value.as_bytes().try_into()
+ }
+}
+
+impl TryFrom<&[u8]> for SourcesFile {
+ type Error = Error;
+
+ fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+ let deserialized = <Vec<SourcesFileRaw>>::deserialize(Deserializer::new(value))?;
+ deserialized.try_into()
+ }
+}
+
+impl TryFrom<Vec<SourcesFileRaw>> for SourcesFile {
+ type Error = Error;
+
+ fn try_from(value: Vec<SourcesFileRaw>) -> Result<Self, Self::Error> {
+ let mut source_packages = Vec::with_capacity(value.len());
+ for entry in value {
+ let entry: SourcePackageEntry = entry.try_into()?;
+ source_packages.push(entry);
+ }
+
+ Ok(Self { source_packages })
+ }
+}
+
+fn parse_file_reference(
+ line: &str,
+ csum_len: usize,
+) -> Result<(String, usize, Vec<u8>), Error> {
+ let mut split = line.split_ascii_whitespace();
+
+ let checksum = split
+ .next()
+ .ok_or_else(|| format_err!("Missing 'checksum' field."))?;
+ if checksum.len() > csum_len * 2 {
+ bail!(
+ "invalid checksum length: '{}', expected {} bytes",
+ checksum,
+ csum_len
+ );
+ }
+
+ let checksum = hex::decode(checksum)?;
+
+ let size = split
+ .next()
+ .ok_or_else(|| format_err!("Missing 'size' field."))?
+ .parse::<usize>()?;
+
+ let file = split
+ .next()
+ .ok_or_else(|| format_err!("Missing 'file name' field."))?
+ .to_string();
+
+ Ok((file, size, checksum))
+}
+
+#[test]
+pub fn test_deb_packages_file() {
+ let input = include_str!(concat!(
+ env!("CARGO_MANIFEST_DIR"),
+ "/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources"
+ ));
+
+ let deserialized =
+ <Vec<SourcesFileRaw>>::deserialize(Deserializer::new(input.as_bytes())).unwrap();
+ assert_eq!(deserialized.len(), 30953);
+
+ let parsed: SourcesFile = deserialized.try_into().unwrap();
+
+ assert_eq!(parsed.source_packages.len(), 30953);
+
+ let found = parsed.source_packages.iter().find(|source| source.package == "base-files").expect("test file contains 'base-files' entry");
+ assert_eq!(found.package, "base-files");
+ assert_eq!(found.format, "3.0 (native)");
+ assert_eq!(found.architecture.as_deref(), Some("any"));
+ assert_eq!(found.directory, "pool/main/b/base-files");
+ assert_eq!(found.section.as_deref(), Some("admin"));
+ assert_eq!(found.version, "11.1+deb11u5");
+
+ let binary_packages = found.binary.as_ref().expect("base-files source package builds base-files binary package");
+ assert_eq!(binary_packages.len(), 1);
+ assert_eq!(binary_packages[0], "base-files");
+
+ let references = &found.files;
+ assert_eq!(references.len(), 2);
+
+ let dsc_file = "base-files_11.1+deb11u5.dsc";
+ let dsc = references.get(dsc_file).expect("base-files source package contains 'dsc' reference");
+ assert_eq!(dsc.file, dsc_file);
+ assert_eq!(dsc.size, 1110);
+ assert_eq!(dsc.checksums.md5.expect("dsc has md5 checksum"), hex::decode("741c34ac0151262a03de8d5a07bc4271").unwrap()[..]);
+ assert_eq!(dsc.checksums.sha256.expect("dsc has sha256 checksum"), hex::decode("c41a7f00d57759f27e6068240d1ea7ad80a9a752e4fb43850f7e86e967422bd3").unwrap()[..]);
+
+ let tar_file = "base-files_11.1+deb11u5.tar.xz";
+ let tar = references.get(tar_file).expect("base-files source package contains 'tar' reference");
+ assert_eq!(tar.file, tar_file);
+ assert_eq!(tar.size, 65612);
+ assert_eq!(tar.checksums.md5.expect("tar has md5 checksum"), hex::decode("995df33642118b566a4026410e1c6aac").unwrap()[..]);
+ assert_eq!(tar.checksums.sha256.expect("tar has sha256 checksum"), hex::decode("31c9e5745845a73f3d5c8a7868c379d77aaca42b81194679d7ab40cc28e3a0e9").unwrap()[..]);
+}
\ No newline at end of file
diff --git a/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources b/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources
new file mode 100644
index 0000000..2b8e387
--- /dev/null
+++ b/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources
@@ -0,0 +1,1 @@
+DOWNLOAD-ME-FROM: http://snapshot.debian.org/archive/debian/20221017T212657Z/dists/bullseye/main/source/Sources.xz
--
2.30.2
More information about the pve-devel
mailing list