[pbs-devel] [PATCH v2 vma-to-pbs] Initial commit

Filip Schauer f.schauer at proxmox.com
Wed Oct 4 10:18:56 CEST 2023


This got superseded by a v3:
https://lists.proxmox.com/pipermail/pve-devel/2023-October/059319.html

On 29/09/2023 15:14, Filip Schauer wrote:
> Implement a tool to import VMA files into a Proxmox Backup Server
>
> Signed-off-by: Filip Schauer <f.schauer at proxmox.com>
> ---
> Changes since v1:
> * Remove unused crates and uses
> * Format the code
> * Use anyhow for error handling
> * Use clap for parsing arguments instead of getopts
> * Fix blocks being reindexed on every read
> * Make sure ProxmoxBackupHandle is dropped properly on error
> * Move image_chunk_buffer from stack to heap
> * Move the block_index in VmaReader to the heap completely
> * Initialize vectors with `Vec::with_capacity` and `resize` instead of
>    the `vec!` macro, to potentially improve performance on debug builds.
> * Add comments to code filling the MD5 sum field with zeros
> * Change device_id arguments to usize
> * Handle devices that have a size that is not aligned to 4096 properly
>    in read_device_contents, when the caller provides a buffer that would
>    exceed the device size.
> * Avoid unnecessary loop iterations in read_device_contents when the
>    buffer size is not aligned to 65536
>
>   .cargo/config                  |   5 +
>   .gitmodules                    |   6 +
>   Cargo.toml                     |  18 ++
>   Makefile                       |  70 +++++++
>   src/main.rs                    | 311 ++++++++++++++++++++++++++++++
>   src/vma.rs                     | 340 +++++++++++++++++++++++++++++++++
>   submodules/proxmox             |   1 +
>   submodules/proxmox-backup-qemu |   1 +
>   8 files changed, 752 insertions(+)
>   create mode 100644 .cargo/config
>   create mode 100644 .gitmodules
>   create mode 100644 Cargo.toml
>   create mode 100644 Makefile
>   create mode 100644 src/main.rs
>   create mode 100644 src/vma.rs
>   create mode 160000 submodules/proxmox
>   create mode 160000 submodules/proxmox-backup-qemu
>
> diff --git a/.cargo/config b/.cargo/config
> new file mode 100644
> index 0000000..3b5b6e4
> --- /dev/null
> +++ b/.cargo/config
> @@ -0,0 +1,5 @@
> +[source]
> +[source.debian-packages]
> +directory = "/usr/share/cargo/registry"
> +[source.crates-io]
> +replace-with = "debian-packages"
> diff --git a/.gitmodules b/.gitmodules
> new file mode 100644
> index 0000000..526f5ef
> --- /dev/null
> +++ b/.gitmodules
> @@ -0,0 +1,6 @@
> +[submodule "submodules/proxmox-backup-qemu"]
> +	path = submodules/proxmox-backup-qemu
> +	url = git://git.proxmox.com/git/proxmox-backup-qemu.git
> +[submodule "submodules/proxmox"]
> +	path = submodules/proxmox
> +	url = git://git.proxmox.com/git/proxmox.git
> diff --git a/Cargo.toml b/Cargo.toml
> new file mode 100644
> index 0000000..ecf9c03
> --- /dev/null
> +++ b/Cargo.toml
> @@ -0,0 +1,18 @@
> +[package]
> +name = "vma-to-pbs"
> +version = "0.0.1"
> +authors = ["Filip Schauer <f.schauer at proxmox.com>"]
> +edition = "2021"
> +
> +[dependencies]
> +anyhow = "1.0"
> +bincode = "1.3"
> +clap = { version = "4.0.32", features = ["cargo"] }
> +md5 = "0.7.0"
> +scopeguard = "1.1.0"
> +serde = "1.0"
> +serde-big-array = "0.4.1"
> +
> +proxmox-backup-qemu = { path = "submodules/proxmox-backup-qemu" }
> +proxmox-io = { path = "submodules/proxmox/proxmox-io" }
> +proxmox-sys = { path = "submodules/proxmox/proxmox-sys" }
> diff --git a/Makefile b/Makefile
> new file mode 100644
> index 0000000..a0c841d
> --- /dev/null
> +++ b/Makefile
> @@ -0,0 +1,70 @@
> +include /usr/share/dpkg/default.mk
> +
> +PACKAGE = proxmox-vma-to-pbs
> +BUILDDIR = $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
> +
> +ARCH := $(DEB_BUILD_ARCH)
> +
> +DSC=$(DEB_SOURCE)_$(DEB_VERSION).dsc
> +MAIN_DEB=$(PACKAGE)_$(DEB_VERSION)_$(ARCH).deb
> +OTHER_DEBS = \
> +	$(PACKAGE)-dev_$(DEB_VERSION)_$(ARCH).deb \
> +	$(PACKAGE)-dbgsym_$(DEB_VERSION)_$(ARCH).deb
> +DEBS=$(MAIN_DEB) $(OTHER_DEBS)
> +
> +DESTDIR=
> +
> +TARGET_DIR := target/debug
> +
> +ifeq ($(BUILD_MODE), release)
> +CARGO_BUILD_ARGS += --release
> +TARGETDIR := target/release
> +endif
> +
> +.PHONY: all build
> +all: build
> +
> +build: $(TARGETDIR)/vma-to-pbs
> +$(TARGETDIR)/vma-to-pbs: Cargo.toml src/
> +	cargo build $(CARGO_BUILD_ARGS)
> +
> +.PHONY: install
> +install: $(TARGETDIR)/vma-to-pbs
> +	install -D -m 0755 $(TARGETDIR)/vma-to-pbs $(DESTDIR)/usr/bin/vma-to-pbs
> +
> +$(BUILDDIR): submodule
> +	rm -rf $@ $@.tmp && mkdir $@.tmp
> +	cp -a submodules debian Makefile .cargo Cargo.toml build.rs src $@.tmp/
> +	mv $@.tmp $@
> +
> +submodule:
> +	[ -e submodules/proxmox-backup-qemu/Cargo.toml ] || [ -e submodules/proxmox/proxmox-sys/Cargo.toml ] || git submodule update --init --recursive
> +
> +dsc:
> +	rm -rf $(BUILDDIR) $(DSC)
> +	$(MAKE) $(DSC)
> +	lintian $(DSC)
> +
> +$(DSC): $(BUILDDIR)
> +	cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d
> +
> +sbuild: $(DSC)
> +	sbuild $<
> +
> +.PHONY: deb dsc
> +deb: $(OTHER_DEBS)
> +$(OTHER_DEBS): $(MAIN_DEB)
> +$(MAIN_DEB): $(BUILDDIR)
> +	cd $(BUILDDIR); dpkg-buildpackage -b -us -uc
> +	lintian $(DEBS)
> +
> +distclean: clean
> +clean:
> +	cargo clean
> +	rm -rf $(PACKAGE)-[0-9]*/
> +	rm -r *.deb *.dsc $(DEB_SOURCE)*.tar* *.build *.buildinfo *.changes Cargo.lock
> +
> +.PHONY: dinstall
> +dinstall: $(DEBS)
> +	dpkg -i $(DEBS)
> +
> diff --git a/src/main.rs b/src/main.rs
> new file mode 100644
> index 0000000..1aefd29
> --- /dev/null
> +++ b/src/main.rs
> @@ -0,0 +1,311 @@
> +extern crate anyhow;
> +extern crate clap;
> +extern crate proxmox_backup_qemu;
> +extern crate proxmox_io;
> +extern crate proxmox_sys;
> +extern crate scopeguard;
> +
> +use std::env;
> +use std::ffi::{c_char, CStr, CString};
> +use std::ptr;
> +use std::time::{SystemTime, UNIX_EPOCH};
> +
> +use anyhow::{anyhow, Context, Result};
> +use clap::{command, Arg, ArgAction};
> +use proxmox_backup_qemu::*;
> +use proxmox_sys::linux::tty;
> +use scopeguard::defer;
> +
> +mod vma;
> +use vma::*;
> +
> +fn backup_vma_to_pbs(
> +    vma_file_path: String,
> +    pbs_repository: String,
> +    backup_id: String,
> +    pbs_password: String,
> +    keyfile: Option<String>,
> +    key_password: Option<String>,
> +    master_keyfile: Option<String>,
> +    fingerprint: String,
> +    compress: bool,
> +    encrypt: bool,
> +) -> Result<()> {
> +    println!("VMA input file: {}", vma_file_path);
> +    println!("PBS repository: {}", pbs_repository);
> +    println!("PBS fingerprint: {}", fingerprint);
> +    println!("compress: {}", compress);
> +    println!("encrypt: {}", encrypt);
> +
> +    let backup_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
> +    println!("backup time: {}", backup_time);
> +
> +    let mut pbs_err: *mut c_char = ptr::null_mut();
> +
> +    let pbs_repository_cstr = CString::new(pbs_repository).unwrap();
> +    let backup_id_cstr = CString::new(backup_id).unwrap();
> +    let pbs_password_cstr = CString::new(pbs_password).unwrap();
> +    let fingerprint_cstr = CString::new(fingerprint).unwrap();
> +    let keyfile_cstr = keyfile.map(|v| CString::new(v).unwrap());
> +    let keyfile_ptr = keyfile_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
> +    let key_password_cstr = key_password.map(|v| CString::new(v).unwrap());
> +    let key_password_ptr = key_password_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
> +    let master_keyfile_cstr = master_keyfile.map(|v| CString::new(v).unwrap());
> +    let master_keyfile_ptr = master_keyfile_cstr.map(|v| v.as_ptr()).unwrap_or(ptr::null());
> +
> +    let pbs = proxmox_backup_new_ns(
> +        pbs_repository_cstr.as_ptr(),
> +        ptr::null(),
> +        backup_id_cstr.as_ptr(),
> +        backup_time,
> +        PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE,
> +        pbs_password_cstr.as_ptr(),
> +        keyfile_ptr,
> +        key_password_ptr,
> +        master_keyfile_ptr,
> +        true,
> +        false,
> +        fingerprint_cstr.as_ptr(),
> +        &mut pbs_err,
> +    );
> +
> +    defer! {
> +        proxmox_backup_disconnect(pbs);
> +    }
> +
> +    if pbs == ptr::null_mut() {
> +        unsafe {
> +            let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +            return Err(anyhow!("proxmox_backup_new_ns failed: {pbs_err_cstr:?}"));
> +        }
> +    }
> +
> +    let connect_result = proxmox_backup_connect(pbs, &mut pbs_err);
> +
> +    if connect_result < 0 {
> +        unsafe {
> +            let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +            return Err(anyhow!("proxmox_backup_connect failed: {pbs_err_cstr:?}"));
> +        }
> +    }
> +
> +    let mut vma_reader = VmaReader::new(&vma_file_path)?;
> +
> +    // Handle configs
> +    let configs = vma_reader.get_configs();
> +    for (config_name, config_data) in configs {
> +        println!("CFG: size: {} name: {}", config_data.len(), config_name);
> +
> +        let config_name_cstr = CString::new(config_name).unwrap();
> +
> +        if proxmox_backup_add_config(
> +            pbs,
> +            config_name_cstr.as_ptr(),
> +            config_data.as_ptr(),
> +            config_data.len() as u64,
> +            &mut pbs_err,
> +        ) < 0
> +        {
> +            unsafe {
> +                let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +                return Err(anyhow!(
> +                    "proxmox_backup_add_config failed: {pbs_err_cstr:?}"
> +                ));
> +            }
> +        }
> +    }
> +
> +    // Handle block devices
> +    for device_id in 0..255 {
> +        let device_name = match vma_reader.get_device_name(device_id) {
> +            Some(x) => x,
> +            None => {
> +                continue;
> +            }
> +        };
> +
> +        let device_size = match vma_reader.get_device_size(device_id) {
> +            Some(x) => x,
> +            None => {
> +                continue;
> +            }
> +        };
> +
> +        println!(
> +            "DEV: dev_id={} size: {} devname: {}",
> +            device_id, device_size, device_name
> +        );
> +
> +        let device_name_cstr = CString::new(device_name).unwrap();
> +        let pbs_device_id = proxmox_backup_register_image(
> +            pbs,
> +            device_name_cstr.as_ptr(),
> +            device_size,
> +            false,
> +            &mut pbs_err,
> +        );
> +
> +        if pbs_device_id < 0 {
> +            unsafe {
> +                let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +                return Err(anyhow!(
> +                    "proxmox_backup_register_image failed: {pbs_err_cstr:?}"
> +                ));
> +            }
> +        }
> +
> +        let mut image_chunk_buffer = proxmox_io::boxed::zeroed(PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE as usize);
> +        let mut bytes_transferred = 0;
> +
> +        while bytes_transferred < device_size {
> +            let bytes_left = device_size - bytes_transferred;
> +            let chunk_size = bytes_left.min(PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE);
> +            println!(
> +                "Uploading dev_id: {} offset: {:#0X} - {:#0X}",
> +                device_id,
> +                bytes_transferred,
> +                bytes_transferred + chunk_size
> +            );
> +
> +            let is_zero_chunk = vma_reader
> +                .read_device_contents(
> +                    device_id,
> +                    &mut image_chunk_buffer[0..chunk_size as usize],
> +                    bytes_transferred,
> +                )
> +                .with_context(|| {
> +                    format!(
> +                        "read {} bytes at offset {} from disk {} from VMA file",
> +                        chunk_size, bytes_transferred, device_id
> +                    )
> +                })?;
> +
> +            let write_data_result = proxmox_backup_write_data(
> +                pbs,
> +                pbs_device_id as u8,
> +                if is_zero_chunk {
> +                    ptr::null()
> +                } else {
> +                    image_chunk_buffer.as_ptr()
> +                },
> +                bytes_transferred,
> +                chunk_size,
> +                &mut pbs_err,
> +            );
> +
> +            if write_data_result < 0 {
> +                unsafe {
> +                    let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +                    return Err(anyhow!(
> +                        "proxmox_backup_write_data failed: {pbs_err_cstr:?}"
> +                    ));
> +                }
> +            }
> +
> +            bytes_transferred += chunk_size;
> +        }
> +
> +        if proxmox_backup_close_image(pbs, pbs_device_id as u8, &mut pbs_err) < 0 {
> +            unsafe {
> +                let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +                return Err(anyhow!(
> +                    "proxmox_backup_close_image failed: {pbs_err_cstr:?}"
> +                ));
> +            }
> +        }
> +    }
> +
> +    if proxmox_backup_finish(pbs, &mut pbs_err) < 0 {
> +        unsafe {
> +            let pbs_err_cstr = CStr::from_ptr(pbs_err);
> +            return Err(anyhow!("proxmox_backup_finish failed: {pbs_err_cstr:?}"));
> +        }
> +    }
> +
> +    Ok(())
> +}
> +
> +fn main() -> Result<()> {
> +    let matches = command!()
> +        .arg(
> +            Arg::new("repository")
> +                .long("repository")
> +                .value_name("auth_id at host:port:datastore")
> +                .help("Repository URL")
> +                .required(true),
> +        )
> +        .arg(
> +            Arg::new("vmid")
> +                .long("vmid")
> +                .value_name("VMID")
> +                .help("Backup ID")
> +                .required(true),
> +        )
> +        .arg(
> +            Arg::new("fingerprint")
> +                .long("fingerprint")
> +                .value_name("FINGERPRINT")
> +                .help("Proxmox Backup Server Fingerprint")
> +                .required(true),
> +        )
> +        .arg(
> +            Arg::new("keyfile")
> +                .long("keyfile")
> +                .value_name("KEYFILE")
> +                .help("Key file"),
> +        )
> +        .arg(
> +            Arg::new("master_keyfile")
> +                .long("master_keyfile")
> +                .value_name("MASTER_KEYFILE")
> +                .help("Master key file"),
> +        )
> +        .arg(
> +            Arg::new("compress")
> +                .long("compress")
> +                .short('c')
> +                .help("Compress the Backup")
> +                .action(ArgAction::SetTrue),
> +        )
> +        .arg(
> +            Arg::new("encrypt")
> +                .long("encrypt")
> +                .short('e')
> +                .help("Encrypt the Backup")
> +                .action(ArgAction::SetTrue),
> +        )
> +        .arg(Arg::new("vma_file"))
> +        .get_matches();
> +
> +    let pbs_repository = matches.get_one::<String>("repository").unwrap().to_string();
> +    let vmid = matches.get_one::<String>("vmid").unwrap().to_string();
> +    let fingerprint = matches.get_one::<String>("fingerprint").unwrap().to_string();
> +
> +    let keyfile = matches.get_one::<String>("keyfile");
> +    let master_keyfile = matches.get_one::<String>("master_keyfile");
> +    let compress = matches.get_flag("compress");
> +    let encrypt = matches.get_flag("encrypt");
> +
> +    let vma_file_path = matches.get_one::<String>("vma_file").unwrap().to_string();
> +
> +    let pbs_password = String::from_utf8(tty::read_password(&"Password: ").unwrap()).unwrap();
> +    let key_password = match keyfile {
> +        Some(_) => Some(String::from_utf8(tty::read_password(&"Key Password: ").unwrap()).unwrap()),
> +        None => None,
> +    };
> +
> +    backup_vma_to_pbs(
> +        vma_file_path,
> +        pbs_repository,
> +        vmid,
> +        pbs_password,
> +        keyfile.cloned(),
> +        key_password,
> +        master_keyfile.cloned(),
> +        fingerprint,
> +        compress,
> +        encrypt,
> +    )?;
> +
> +    Ok(())
> +}
> diff --git a/src/vma.rs b/src/vma.rs
> new file mode 100644
> index 0000000..e2c3475
> --- /dev/null
> +++ b/src/vma.rs
> @@ -0,0 +1,340 @@
> +extern crate anyhow;
> +extern crate md5;
> +
> +use std::collections::HashMap;
> +use std::fs::File;
> +use std::io::{Read, Seek, SeekFrom};
> +use std::mem::size_of;
> +use std::{cmp, str};
> +
> +use anyhow::{anyhow, Result};
> +use bincode::Options;
> +use serde::{Deserialize, Serialize};
> +use serde_big_array::BigArray;
> +
> +const VMA_BLOCKS_PER_EXTENT: usize = 59;
> +const VMA_MAX_CONFIGS: usize = 256;
> +const VMA_MAX_DEVICES: usize = 256;
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaDeviceInfoHeader {
> +    pub device_name_offset: u32,
> +    reserved: [u8; 4],
> +    pub device_size: u64,
> +    reserved1: [u8; 16],
> +}
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaHeader {
> +    pub magic: [u8; 4],
> +    pub version: u32,
> +    pub uuid: [u8; 16],
> +    pub ctime: u64,
> +    pub md5sum: [u8; 16],
> +    pub blob_buffer_offset: u32,
> +    pub blob_buffer_size: u32,
> +    pub header_size: u32,
> +    #[serde(with = "BigArray")]
> +    reserved: [u8; 1984],
> +    #[serde(with = "BigArray")]
> +    pub config_names: [u32; VMA_MAX_CONFIGS],
> +    #[serde(with = "BigArray")]
> +    pub config_data: [u32; VMA_MAX_CONFIGS],
> +    reserved1: [u8; 4],
> +    #[serde(with = "BigArray")]
> +    pub dev_info: [VmaDeviceInfoHeader; VMA_MAX_DEVICES],
> +}
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaBlockInfo {
> +    pub mask: u16,
> +    reserved: u8,
> +    pub dev_id: u8,
> +    pub cluster_num: u32,
> +}
> +
> +#[repr(C)]
> +#[derive(Serialize, Deserialize)]
> +struct VmaExtentHeader {
> +    pub magic: [u8; 4],
> +    reserved: [u8; 2],
> +    pub block_count: u16,
> +    pub uuid: [u8; 16],
> +    pub md5sum: [u8; 16],
> +    #[serde(with = "BigArray")]
> +    pub blockinfo: [VmaBlockInfo; VMA_BLOCKS_PER_EXTENT],
> +}
> +
> +#[derive(Clone)]
> +struct VmaBlockIndexEntry {
> +    pub cluster_file_offset: u64,
> +    pub mask: u16,
> +}
> +
> +pub struct VmaReader {
> +    vma_file: File,
> +    vma_header: VmaHeader,
> +    configs: HashMap<String, String>,
> +    block_index: Vec<Vec<VmaBlockIndexEntry>>,
> +    blocks_are_indexed: bool,
> +}
> +
> +impl VmaReader {
> +    pub fn new(vma_file_path: &str) -> Result<Self> {
> +        let mut vma_file = match File::open(vma_file_path) {
> +            Err(why) => return Err(anyhow!("couldn't open {}: {}", vma_file_path, why)),
> +            Ok(file) => file,
> +        };
> +
> +        let vma_header = Self::read_header(&mut vma_file).unwrap();
> +        let configs = Self::read_blob_buffer(&mut vma_file, &vma_header).unwrap();
> +        let block_index: Vec<Vec<VmaBlockIndexEntry>> = (0..256).map(|_| Vec::new()).collect();
> +
> +        let instance = Self {
> +            vma_file,
> +            vma_header,
> +            configs,
> +            block_index,
> +            blocks_are_indexed: false,
> +        };
> +
> +        Ok(instance)
> +    }
> +
> +    fn read_header(vma_file: &mut File) -> Result<VmaHeader> {
> +        let mut buffer = Vec::with_capacity(size_of::<VmaHeader>());
> +        buffer.resize(size_of::<VmaHeader>(), 0);
> +        vma_file.read_exact(&mut buffer)?;
> +
> +        let bincode_options = bincode::DefaultOptions::new()
> +            .with_fixint_encoding()
> +            .with_big_endian();
> +
> +        let vma_header: VmaHeader = bincode_options.deserialize(&buffer)?;
> +
> +        if vma_header.magic != [b'V', b'M', b'A', 0] {
> +            return Err(anyhow!("Invalid magic number"));
> +        }
> +
> +        if vma_header.version != 1 {
> +            return Err(anyhow!("Invalid VMA version {}", vma_header.version));
> +        }
> +
> +        buffer.resize(vma_header.header_size as usize, 0);
> +        vma_file.read_exact(&mut buffer[size_of::<VmaHeader>()..])?;
> +
> +        // Fill the MD5 sum field with zeros to compute the MD5 sum
> +        buffer[32..48].fill(0);
> +        let computed_md5sum: [u8; 16] = md5::compute(&buffer).into();
> +
> +        if vma_header.md5sum != computed_md5sum {
> +            return Err(anyhow!("Wrong VMA header checksum"));
> +        }
> +
> +        return Ok(vma_header);
> +    }
> +
> +    fn read_string_from_file(vma_file: &mut File, file_offset: u64) -> Result<String> {
> +        let mut size_bytes = [0u8; 2];
> +        vma_file.seek(SeekFrom::Start(file_offset))?;
> +        vma_file.read_exact(&mut size_bytes)?;
> +        let size = u16::from_le_bytes(size_bytes) as usize;
> +        let mut string_bytes = Vec::with_capacity(size - 1);
> +        string_bytes.resize(size - 1, 0);
> +        vma_file.read_exact(&mut string_bytes)?;
> +        let string = str::from_utf8(&string_bytes)?;
> +
> +        return Ok(string.to_string());
> +    }
> +
> +    fn read_blob_buffer(
> +        vma_file: &mut File,
> +        vma_header: &VmaHeader,
> +    ) -> Result<HashMap<String, String>> {
> +        let mut configs = HashMap::new();
> +
> +        for i in 0..VMA_MAX_CONFIGS {
> +            let config_name_offset = vma_header.config_names[i];
> +            let config_data_offset = vma_header.config_data[i];
> +
> +            if config_name_offset == 0 || config_data_offset == 0 {
> +                continue;
> +            }
> +
> +            let config_name_file_offset = (vma_header.blob_buffer_offset + config_name_offset) as u64;
> +            let config_data_file_offset = (vma_header.blob_buffer_offset + config_data_offset) as u64;
> +            let config_name = Self::read_string_from_file(vma_file, config_name_file_offset)?;
> +            let config_data = Self::read_string_from_file(vma_file, config_data_file_offset)?;
> +
> +            configs.insert(String::from(config_name), String::from(config_data));
> +        }
> +
> +        return Ok(configs);
> +    }
> +
> +    pub fn get_configs(&self) -> HashMap<String, String> {
> +        return self.configs.clone();
> +    }
> +
> +    pub fn get_device_name(&mut self, device_id: usize) -> Option<String> {
> +        if device_id >= VMA_MAX_DEVICES {
> +            return None;
> +        }
> +
> +        let device_name_offset = self.vma_header.dev_info[device_id].device_name_offset;
> +
> +        if device_name_offset == 0 {
> +            return None;
> +        }
> +
> +        let device_name_file_offset = (self.vma_header.blob_buffer_offset + device_name_offset) as u64;
> +        let device_name = Self::read_string_from_file(&mut self.vma_file, device_name_file_offset).unwrap();
> +
> +        return Some(device_name.to_string());
> +    }
> +
> +    pub fn get_device_size(&self, device_id: usize) -> Option<u64> {
> +        if device_id >= VMA_MAX_DEVICES {
> +            return None;
> +        }
> +
> +        let dev_info = &self.vma_header.dev_info[device_id];
> +
> +        if dev_info.device_name_offset == 0 {
> +            return None;
> +        }
> +
> +        return Some(dev_info.device_size);
> +    }
> +
> +    fn read_extent_header(vma_file: &mut File) -> Result<VmaExtentHeader> {
> +        let mut buffer = Vec::with_capacity(size_of::<VmaExtentHeader>());
> +        buffer.resize(size_of::<VmaExtentHeader>(), 0);
> +        vma_file.read_exact(&mut buffer)?;
> +
> +        let bincode_options = bincode::DefaultOptions::new()
> +            .with_fixint_encoding()
> +            .with_big_endian();
> +
> +        let vma_extent_header: VmaExtentHeader = bincode_options.deserialize(&buffer)?;
> +
> +        if vma_extent_header.magic != [b'V', b'M', b'A', b'E'] {
> +            return Err(anyhow!("Invalid magic number"));
> +        }
> +
> +        // Fill the MD5 sum field with zeros to compute the MD5 sum
> +        buffer[24..40].fill(0);
> +        let computed_md5sum: [u8; 16] = md5::compute(&buffer).into();
> +
> +        if vma_extent_header.md5sum != computed_md5sum {
> +            return Err(anyhow!("Wrong VMA extent header checksum"));
> +        }
> +
> +        return Ok(vma_extent_header);
> +    }
> +
> +    fn index_device_clusters(&mut self) -> Result<()> {
> +        for device_id in 0..255 {
> +            let device_size = match self.get_device_size(device_id) {
> +                Some(x) => x,
> +                None => {
> +                    continue;
> +                }
> +            };
> +
> +            let device_cluster_count = (device_size + 4096 * 16 - 1) / (4096 * 16);
> +
> +            let block_index_entry_placeholder = VmaBlockIndexEntry {
> +                cluster_file_offset: 0,
> +                mask: 0,
> +            };
> +
> +            self.block_index[device_id].resize(device_cluster_count as usize, block_index_entry_placeholder);
> +        }
> +
> +        let mut file_offset = self.vma_header.header_size as u64;
> +        let vma_file_size = self.vma_file.metadata()?.len();
> +
> +        while file_offset < vma_file_size {
> +            self.vma_file.seek(SeekFrom::Start(file_offset))?;
> +            let vma_extent_header = Self::read_extent_header(&mut self.vma_file)?;
> +            file_offset += size_of::<VmaExtentHeader>() as u64;
> +
> +            for i in 0..VMA_BLOCKS_PER_EXTENT {
> +                let blockinfo = &vma_extent_header.blockinfo[i];
> +
> +                if blockinfo.dev_id == 0 {
> +                    continue;
> +                }
> +
> +                let block_index_entry = VmaBlockIndexEntry {
> +                    cluster_file_offset: file_offset,
> +                    mask: blockinfo.mask,
> +                };
> +
> +                self.block_index[blockinfo.dev_id as usize][blockinfo.cluster_num as usize] = block_index_entry;
> +                file_offset += blockinfo.mask.count_ones() as u64 * 4096;
> +            }
> +        }
> +
> +        self.blocks_are_indexed = true;
> +
> +        return Ok(());
> +    }
> +
> +    pub fn read_device_contents(
> +        &mut self,
> +        device_id: usize,
> +        buffer: &mut [u8],
> +        offset: u64,
> +    ) -> Result<bool> {
> +        if device_id >= VMA_MAX_DEVICES {
> +            return Err(anyhow!("invalid device id {}", device_id));
> +        }
> +
> +        if offset % (4096 * 16) != 0 {
> +            return Err(anyhow!("offset is not aligned to 65536"));
> +        }
> +
> +        // Make sure that the device clusters are already indexed
> +        if !self.blocks_are_indexed {
> +            self.index_device_clusters()?;
> +        }
> +
> +        let this_device_block_index = &self.block_index[device_id];
> +        let length = cmp::min(
> +            buffer.len(),
> +            this_device_block_index.len() * 4096 * 16 - offset as usize,
> +        );
> +        let mut buffer_offset = 0;
> +        let mut buffer_is_zero = true;
> +
> +        while buffer_offset < length {
> +            let block_index_entry = &this_device_block_index[(offset as usize + buffer_offset) / (4096 * 16)];
> +            self.vma_file.seek(SeekFrom::Start(block_index_entry.cluster_file_offset))?;
> +
> +            for i in 0..16 {
> +                if buffer_offset >= length {
> +                    break;
> +                }
> +
> +                let block_buffer_end = buffer_offset + cmp::min(length - buffer_offset, 4096);
> +                let block_mask = ((block_index_entry.mask >> i) & 1) == 1;
> +
> +                if block_mask {
> +                    self.vma_file.read_exact(&mut buffer[buffer_offset..block_buffer_end])?;
> +                    buffer_is_zero = false;
> +                } else {
> +                    buffer[buffer_offset..block_buffer_end].fill(0);
> +                }
> +
> +                buffer_offset += 4096;
> +            }
> +        }
> +
> +        return Ok(buffer_is_zero);
> +    }
> +}
> diff --git a/submodules/proxmox b/submodules/proxmox
> new file mode 160000
> index 0000000..dc9ee73
> --- /dev/null
> +++ b/submodules/proxmox
> @@ -0,0 +1 @@
> +Subproject commit dc9ee737512fc2c7325f47b875d6c69ccf484cea
> diff --git a/submodules/proxmox-backup-qemu b/submodules/proxmox-backup-qemu
> new file mode 160000
> index 0000000..73a09e9
> --- /dev/null
> +++ b/submodules/proxmox-backup-qemu
> @@ -0,0 +1 @@
> +Subproject commit 73a09e96720434e4aba7f876f9c6cf56bce58c2c





More information about the pbs-devel mailing list