[pbs-devel] [PATCH v5 proxmox-backup 52/62] docs: file formats: describe split pxar archive file layout

[Christian Ebner]

Describes the pxar metadata archive and the corresponding pxar payload
file-format layout.

Signed-off-by: Christian Ebner <c.ebner at proxmox.com>
---
changes since version 4:
- no changes

 docs/file-formats.rst         | 46 ++++++++++++++++++++++++++++++++
 docs/meta-format-overview.dot | 50 +++++++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 docs/meta-format-overview.dot

diff --git a/docs/file-formats.rst b/docs/file-formats.rst
index 43ecfefce..77d55b5ef 100644
--- a/docs/file-formats.rst
+++ b/docs/file-formats.rst
@@ -8,7 +8,53 @@ Proxmox File Archive Format (``.pxar``)
 
 .. graphviz:: pxar-format-overview.dot
 
+.. _pxar-meta-format:
 
+Proxmox File Archive Format - Meta (``.mpxar``)
+-----------------------------------------------
+
+Pxar metadata archive with same structure as a regular pxar archive, with the
+exception of regular file payloads not being contained within the archive
+itself, but rather being stored as payload references to the corresponding pxar
+payload (``.ppxar``) file.
+
+Can be used to lookup all the archive entries and metadata without the size
+overhead introduced by the file payloads.
+
+.. graphviz:: meta-format-overview.dot
+
+.. _ppxar-format:
+
+Proxmox File Archive Format - Payload (``.ppxar``)
+--------------------------------------------------
+
+Pxar payload file storing regular file payloads to be referenced and accessed by
+the corresponding pxar metadata (``.mpxar``) archive. Contains a concatenation
+of regular file payloads, each prefixed by a `PAYLOAD` header. Further, the
+actual referenced payload entries might be separated by padding (full/partial
+payloads not referenced), introduced when reusing chunks of a previous backup
+run, when chunk boundaries did not aligned to payload entry offsets.
+
+All headers are stored as little-endian.
+
+.. list-table::
+   :widths: auto
+
+   * - ``PAYLOAD_START_MARKER``
+     - header of ``[u8; 16]`` consisting of type hash and size;
+       marks start
+   * - ``PAYLOAD``
+     - header of ``[u8; 16]`` cosisting of type hash and size;
+       referenced by metadata archive
+   * - Payload
+     - raw regular file payload
+   * - Padding
+     - partial/full unreferenced payloads, caused by unaligned chunk boundary
+   * - ...
+     - further concatenation of payload header, payload and padding
+   * - ``PAYLOAD_TAIL_MARKER``
+     - header of ``[u8; 16]`` consisting of type hash and size;
+       marks end
 .. _data-blob-format:
 
 Data Blob Format (``.blob``)
diff --git a/docs/meta-format-overview.dot b/docs/meta-format-overview.dot
new file mode 100644
index 000000000..7eea4b55b
--- /dev/null
+++ b/docs/meta-format-overview.dot
@@ -0,0 +1,50 @@
+digraph g {
+graph [
+rankdir = "LR"
+fontname="Helvetica"
+];
+node [
+fontsize = "16"
+shape = "record"
+];
+edge [
+];
+
+"archive" [
+label = "archive.mpxar"
+shape = "record"
+];
+
+"rootdir" [
+label = "<fv>FORMAT_VERSION\l|PRELUDE\l|<f0>ENTRY\l|\{XATTR\}\* extended attribute list\l|\{ACL_USER\}\* USER ACL entries\l|\{ACL_GROUP\}\* GROUP ACL entries\l|\[ACL_GROUP_OBJ\] the ACL_GROUP_OBJ \l|\[ACL_DEFAULT\] the various default ACL fields\l|\{ACL_DEFAULT_USER\}\* USER ACL entries\l|\{ACL_DEFAULT_GROUP\}\* GROUP ACL entries\l|\[FCAPS\] file capability in Linux disk format\l|\[QUOTA_PROJECT_ID\] the ext4/xfs quota project ID\l|{<pl> PAYLOAD_REF|SYMLINK|DEVICE|{<de> \{DirectoryEntries\}\*|GOODBYE}}"
+shape = "record"
+];
+
+
+"entry" [
+label = "<f0> size: u64 = 64\l|type: u64 = ENTRY\l|feature_flags: u64\l|mode: u64\l|flags: u64\l|uid: u64\l|gid: u64\l|mtime: u64\l"
+labeljust = "l"
+shape = "record"
+];
+
+
+
+"direntry" [
+label = "<f0> FILENAME\l|{ENTRY\l|HARDLINK\l}"
+shape = "record"
+];
+
+"payloadrefentry" [
+label = "<f0> offset: u64\l|size: u64\l"
+shape = "record"
+];
+
+"archive" -> "rootdir":fv
+
+"rootdir":f0 -> "entry":f0
+
+"rootdir":de -> "direntry":f0
+
+"rootdir":pl -> "payloadrefentry":f0
+
+}
-- 
2.39.2