[pbs-devel] [PATCH v6 proxmox-backup 64/65] chunker: tests: add regression tests for payload chunker

[Dominik Csapak]

these tests fail here with:

---- chunker::test_suggested_boundary stdout ----
Chunk at 1, total 32768 from base 0
Chunk at 1, total 143377 from base 0
Chunk at 1, total 405521 from base 0
Chunk at 1, total 667665 from base 0
Chunk at 1, total 929809 from base 0
thread 'chunker::test_suggested_boundary' panicked at 'attempt to subtract with overflow', 
pbs-datastore/src/chunker.rs:253:19
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

is there something wrong in the code or in the tests (or in my environment) ?


On 5/14/24 12:34, Christian Ebner wrote:
> Test chunking of a payload stream with suggested chunk boundaries.
> 
> Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
> ---
>   pbs-datastore/src/chunker.rs | 92 ++++++++++++++++++++++++++++++++++++
>   1 file changed, 92 insertions(+)
> 
> diff --git a/pbs-datastore/src/chunker.rs b/pbs-datastore/src/chunker.rs
> index bfa1c8ca1..ad91a1599 100644
> --- a/pbs-datastore/src/chunker.rs
> +++ b/pbs-datastore/src/chunker.rs
> @@ -381,3 +381,95 @@ fn test_chunker1() {
>           panic!("got different chunks");
>       }
>   }
> +
> +#[test]
> +fn test_suggested_boundary() {
> +    let mut buffer = Vec::new();
> +
> +    for i in 0..(256 * 1024) {
> +        for j in 0..4 {
> +            let byte = ((i >> (j << 3)) & 0xff) as u8;
> +            buffer.push(byte);
> +        }
> +    }
> +    let (tx, rx) = std::sync::mpsc::channel();
> +    let mut chunker = PayloadChunker::new(64 * 1024, rx);
> +
> +    // Suggest chunk boundary within regular chunk
> +    tx.send(32 * 1024).unwrap();
> +    // Suggest chunk boundary within regular chunk, resulting chunk being 0
> +    tx.send(32 * 1024).unwrap();
> +    // Suggest chunk boundary in the past, must be ignored
> +    tx.send(0).unwrap();
> +    // Suggest chunk boundary aligned with regular boundary
> +    tx.send(405521).unwrap();
> +
> +    let mut pos = 0;
> +    let mut last = 0;
> +
> +    let mut chunks1: Vec<(usize, usize)> = vec![];
> +    let mut chunks2: Vec<(usize, usize)> = vec![];
> +    let mut ctx = Context::default();
> +
> +    // test1: feed single bytes with suggeset boundary
> +    while pos < buffer.len() {
> +        ctx.total += 1;
> +        let k = chunker.scan(&buffer[pos..pos + 1], &ctx);
> +        pos += 1;
> +        if k != 0 {
> +            println!("Chunk at {k}, total {} from base {}", ctx.total, ctx.base);
> +            let prev = last;
> +            last = pos;
> +            chunks1.push((prev, pos - prev));
> +        }
> +    }
> +    chunks1.push((last, buffer.len() - last));
> +
> +    let mut pos = 0;
> +    let mut ctx = Context::default();
> +    // Suggest chunk boundary within regular chunk
> +    tx.send(32 * 1024).unwrap();
> +    // Suggest chunk boundary within regular chunk,
> +    // resulting chunk being to small and therefore ignored
> +    tx.send(32 * 1024).unwrap();
> +    // Suggest chunk boundary in the past, must be ignored
> +    tx.send(0).unwrap();
> +    // Suggest chunk boundary aligned with regular boundary
> +    tx.send(405521).unwrap();
> +    ctx.total = buffer.len() as u64;
> +
> +    while pos < buffer.len() {
> +        let k = chunker.scan(&buffer[pos..], &ctx);
> +        if k != 0 {
> +            chunks2.push((pos, k));
> +            pos += k;
> +            ctx.base += pos as u64;
> +            ctx.total -= pos as u64;
> +        } else {
> +            break;
> +        }
> +    }
> +
> +    chunks2.push((pos, buffer.len() - pos));
> +
> +    if chunks1 != chunks2 {
> +        let mut size1 = 0;
> +        for (_offset, len) in &chunks1 {
> +            size1 += len;
> +        }
> +        println!("Chunks1: {size1}\n{chunks1:?}\n");
> +
> +        let mut size2 = 0;
> +        for (_offset, len) in &chunks2 {
> +            size2 += len;
> +        }
> +        println!("Chunks2: {size2}\n{chunks2:?}\n");
> +
> +        panic!("got different chunks");
> +    }
> +
> +    let expected_sizes = [32768, 110609, 262144, 262144, 262144, 118767];
> +    for ((_, chunk_size), expected) in chunks1.iter().zip(expected_sizes.iter()) {
> +        assert_eq!(chunk_size, expected);
> +    }
> +}