[pbs-devel] [PATCH v4 proxmox-backup 09/11] datastore: use u64 instead of usize for fidx writer content size
Robert Obkircher
r.obkircher at proxmox.com
Fri Jan 23 16:37:22 CET 2026
This is closer to what the file format supports.
Signed-off-by: Robert Obkircher <r.obkircher at proxmox.com>
---
pbs-datastore/src/datastore.rs | 6 +--
pbs-datastore/src/fixed_index.rs | 69 ++++++++++++++++----------------
src/api2/backup/environment.rs | 6 +--
src/api2/backup/mod.rs | 2 +-
4 files changed, 42 insertions(+), 41 deletions(-)
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 56dfce6e..8770d942 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -695,11 +695,11 @@ impl DataStore {
pub fn create_fixed_writer<P: AsRef<Path>>(
&self,
filename: P,
- size: Option<usize>,
- chunk_size: usize,
+ size: Option<u64>,
+ chunk_size: u32,
) -> Result<FixedIndexWriter, Error> {
let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
- FixedIndexWriter::create(full_path, size, chunk_size)
+ FixedIndexWriter::create(full_path, size, chunk_size.into())
}
pub fn open_fixed_reader<P: AsRef<Path>>(
diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs
index 056ae07b..c2888372 100644
--- a/pbs-datastore/src/fixed_index.rs
+++ b/pbs-datastore/src/fixed_index.rs
@@ -214,8 +214,8 @@ pub struct FixedIndexWriter {
file: File,
filename: PathBuf,
tmp_filename: PathBuf,
- chunk_size: usize,
- size: usize,
+ chunk_size: u64,
+ size: u64,
index_length: usize,
index_capacity: usize,
index: *mut u8,
@@ -248,8 +248,8 @@ impl FixedIndexWriter {
// Requires obtaining a shared chunk store lock beforehand
pub fn create(
full_path: impl Into<PathBuf>,
- known_size: Option<usize>,
- chunk_size: usize,
+ known_size: Option<u64>,
+ chunk_size: u64,
) -> Result<Self, Error> {
let full_path = full_path.into();
let mut tmp_path = full_path.clone();
@@ -287,10 +287,13 @@ impl FixedIndexWriter {
file.write_all(&buffer)?;
- let (index_length, index_capacity) = known_size
- .map(|s| s.div_ceil(chunk_size))
- .map(|len| (len, len))
- .unwrap_or((0, Self::INITIAL_CAPACITY));
+ let (index_length, index_capacity) = match known_size {
+ Some(s) => {
+ let len = s.div_ceil(chunk_size).try_into()?;
+ (len, len)
+ }
+ None => (0, Self::INITIAL_CAPACITY),
+ };
let index_size = index_capacity * 32;
nix::unistd::ftruncate(&file, (header_size + index_size) as i64)?;
@@ -376,13 +379,13 @@ impl FixedIndexWriter {
/// The size also becomes fixed as soon as it is no longer divisible
/// by the block size, to ensure that only the last block can be
/// smaller.
- fn grow_to_size(&mut self, requested_size: usize) -> Result<(), Error> {
+ fn grow_to_size(&mut self, requested_size: u64) -> Result<(), Error> {
if self.size < requested_size {
if !self.growable_size {
bail!("refusing to resize from {} to {requested_size}", self.size);
}
- let new_len = requested_size.div_ceil(self.chunk_size);
- if new_len * self.chunk_size != requested_size {
+ let new_len = requested_size.div_ceil(self.chunk_size).try_into()?;
+ if new_len as u64 * self.chunk_size != requested_size {
// not a full chunk, so this must be the last one
self.growable_size = false;
self.set_index_capacity_or_unmap(new_len)?;
@@ -463,12 +466,10 @@ impl FixedIndexWriter {
Ok(index_csum)
}
- fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result<usize, Error> {
- if offset < chunk_len {
+ fn check_chunk_alignment(&self, offset: u64, chunk_len: u64) -> Result<usize, Error> {
+ let Some(pos) = offset.checked_sub(chunk_len) else {
bail!("got chunk with small offset ({} < {}", offset, chunk_len);
- }
-
- let pos = offset - chunk_len;
+ };
if offset > self.size {
bail!("chunk data exceeds size ({} >= {})", offset, self.size);
@@ -490,7 +491,7 @@ impl FixedIndexWriter {
bail!("got unaligned chunk (pos = {})", pos);
}
- Ok(pos / self.chunk_size)
+ Ok((pos / self.chunk_size) as usize)
}
fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
@@ -524,12 +525,12 @@ impl FixedIndexWriter {
/// If this writer has been created without a fixed size, the
/// index capacity and content size are increased automatically
/// until an incomplete chunk is encountered.
- pub fn add_chunk(&mut self, start: u64, size: u32, digest: &[u8; 32]) -> Result<(), Error> {
- let Some(end) = start.checked_add(size.into()) else {
+ pub fn add_chunk(&mut self, start: u64, size: u64, digest: &[u8; 32]) -> Result<(), Error> {
+ let Some(end) = start.checked_add(size) else {
bail!("add_chunk: start and size are too large: {start}+{size}");
};
- self.grow_to_size(end as usize)?;
- let idx = self.check_chunk_alignment(end as usize, size as usize)?;
+ self.grow_to_size(end)?;
+ let idx = self.check_chunk_alignment(end, size)?;
self.add_digest(idx, digest)
}
@@ -538,7 +539,7 @@ impl FixedIndexWriter {
bail!("reusing the index is only supported with known input size");
}
- if self.chunk_size != reader.chunk_size {
+ if Ok(self.chunk_size) != reader.chunk_size.try_into() {
bail!("can't reuse file with different chunk size");
}
@@ -560,7 +561,7 @@ mod tests {
use std::env;
use std::fs;
- const CS: usize = 4096;
+ const CS: u64 = 4096;
#[test]
fn test_empty() {
@@ -606,7 +607,7 @@ mod tests {
let initial = FixedIndexWriter::INITIAL_CAPACITY;
let steps = [1, 2, initial, initial + 1, 5 * initial, 10 * initial + 1];
- let expected = test_data(steps.last().unwrap() * CS);
+ let expected = test_data(*steps.last().unwrap() as u64 * CS);
let mut begin = 0;
for chunk_count in steps {
@@ -623,7 +624,7 @@ mod tests {
w.close().unwrap();
drop(w);
- let size = expected.len() * CS;
+ let size = expected.len() as u64 * CS;
check_with_reader(&path, size, &expected);
compare_to_known_size_writer(&path, size, &expected);
}
@@ -634,7 +635,7 @@ mod tests {
let path = dir.join("test_grow_to_misaligned_size");
let mut w = FixedIndexWriter::create(&path, None, CS).unwrap();
- let size = (FixedIndexWriter::INITIAL_CAPACITY + 42) * CS - 1; // last is not full
+ let size = (FixedIndexWriter::INITIAL_CAPACITY as u64 + 42) * CS - 1; // last is not full
let expected = test_data(size);
w.grow_to_size(size).unwrap();
@@ -677,8 +678,8 @@ mod tests {
struct TestChunk {
digest: [u8; 32],
index: usize,
- size: usize,
- end: usize,
+ size: u64,
+ end: u64,
}
impl TestChunk {
@@ -691,7 +692,7 @@ mod tests {
}
}
- fn test_data(size: usize) -> Vec<TestChunk> {
+ fn test_data(size: u64) -> Vec<TestChunk> {
(0..size.div_ceil(CS))
.map(|index| {
let mut digest = [0u8; 32];
@@ -706,24 +707,24 @@ mod tests {
};
TestChunk {
digest,
- index,
+ index: index as usize,
size,
- end: index * CS + size,
+ end: index as u64 * CS + size,
}
})
.collect()
}
- fn check_with_reader(path: &Path, size: usize, chunks: &[TestChunk]) {
+ fn check_with_reader(path: &Path, size: u64, chunks: &[TestChunk]) {
let reader = FixedIndexReader::open(path).unwrap();
- assert_eq!(size as u64, reader.index_bytes());
+ assert_eq!(size, reader.index_bytes());
assert_eq!(chunks.len(), reader.index_count());
for c in chunks {
assert_eq!(&c.digest, reader.index_digest(c.index).unwrap());
}
}
- fn compare_to_known_size_writer(file: &Path, size: usize, chunks: &[TestChunk]) {
+ fn compare_to_known_size_writer(file: &Path, size: u64, chunks: &[TestChunk]) {
let mut path = file.to_path_buf();
path.set_extension("reference");
let mut w = FixedIndexWriter::create(&path, Some(size), CS).unwrap();
diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 04c5bf84..7d49d47c 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -67,7 +67,7 @@ struct DynamicWriterState {
struct FixedWriterState {
name: String,
index: FixedIndexWriter,
- size: Option<usize>,
+ size: Option<u64>,
chunk_size: u32,
chunk_count: u64,
small_chunk_count: usize, // allow 0..1 small chunks (last chunk may be smaller)
@@ -349,7 +349,7 @@ impl BackupEnvironment {
&self,
index: FixedIndexWriter,
name: String,
- size: Option<usize>,
+ size: Option<u64>,
chunk_size: u32,
incremental: bool,
) -> Result<usize, Error> {
@@ -442,7 +442,7 @@ impl BackupEnvironment {
);
}
- data.index.add_chunk(offset, size, digest)?;
+ data.index.add_chunk(offset, size.into(), digest)?;
data.chunk_count += 1;
diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs
index c2822c18..54445efa 100644
--- a/src/api2/backup/mod.rs
+++ b/src/api2/backup/mod.rs
@@ -480,7 +480,7 @@ fn create_fixed_index(
let env: &BackupEnvironment = rpcenv.as_ref();
let name = required_string_param(¶m, "archive-name")?.to_owned();
- let size = param["size"].as_u64().map(usize::try_from).transpose()?;
+ let size = param["size"].as_u64();
let reuse_csum = param["reuse-csum"].as_str();
let archive_name = name.clone();
--
2.47.3
More information about the pbs-devel
mailing list