[pve-devel] [PATCH v1 pve-storage 1/2] example: sshfs plugin: add custom storage plugin for SSHFS
Max Carrara
m.carrara at proxmox.com
Fri Apr 18 14:18:52 CEST 2025
On Wed Apr 16, 2025 at 2:47 PM CEST, Max Carrara wrote:
> This commit adds an example implementation of a custom storage plugin
> that uses SSHFS [0] as the underlying filesystem.
>
> The implementation is very similar to that of the NFS plugin; as a
> prerequisite, it is currently necessary to use pubkey auth and have
> the host's root user's public key deployed to the remote host like so:
>
> ssh-copy-id -i ~/.ssh/id_my_private_key \
> -o UserKnownHostsFile=/etc/pve/priv/known_hosts [USER]@[HOST]
>
> Then, the storage can be added as follows:
>
> pvesm add sshfs [STOREID] \
> --username [USER] \
> --server [HOST] \
> --sshfs-remote-path [ABS PATH ON REMOTE] \
> --path /mnt/path/to/storage \
> --sshfs-private-key ~/.ssh/id_my_private_key
>
> If the host is part of a cluster, other nodes may connect to the
> remote without any additional setup required. This is because we copy
> the private key to `/etc/pve/priv/storage/$KEYNAME.key` and use the
> cluster-wide `/etc/pve/priv/known_hosts` file. Also mark each SSHFS
> storage as `shared` by default in order to make use of this.
>
> Note: Because there's currently no way to officially and permanently
> mark a storage as shared (like some built-in plugins [1]) set
> `$scfg->{shared} = 1;` in `on_add_hook`. This has almost the same
> effect as modifying `@PVE::Storage::Plugin::SHARED_STORAGE` directly,
> except that the `shared` is written to `/etc/pve/storage.cfg`.
>
> [0]: https://github.com/libfuse/sshfs
> [1]: https://git.proxmox.com/?p=pve-storage.git;a=blob;f=src/PVE/Storage/Plugin.pm;h=4e16420f667f196e8eb99ae7c9f3f1d3e13791fb;hb=refs/heads/master#l37
>
> Signed-off-by: Max Carrara <m.carrara at proxmox.com>
> ---
> Changes rfc-v1 --> v1:
> * rework most of the plugin
> * cease to call methods of DirPlugin (plugins should be isolated;
> we don't want to encourage third-party devs to do that)
> * handle SSH private key as sensitive property and place it on pmxcfs
> * make storage shared
> * manually implement attribute handling ("notes", "protected" for
> backups)
>
> .../lib/PVE/Storage/Custom/SSHFSPlugin.pm | 398 ++++++++++++++++++
> 1 file changed, 398 insertions(+)
> create mode 100644 example/sshfs-plugin/lib/PVE/Storage/Custom/SSHFSPlugin.pm
>
> diff --git a/example/sshfs-plugin/lib/PVE/Storage/Custom/SSHFSPlugin.pm b/example/sshfs-plugin/lib/PVE/Storage/Custom/SSHFSPlugin.pm
> new file mode 100644
> index 0000000..75b29c1
> --- /dev/null
> +++ b/example/sshfs-plugin/lib/PVE/Storage/Custom/SSHFSPlugin.pm
> @@ -0,0 +1,398 @@
> +package PVE::Storage::Custom::SSHFSPlugin;
> +
> +use strict;
> +use warnings;
> +
> +use feature 'signatures';
> +
> +use Cwd qw();
> +use Encode qw(decode encode);
> +use File::Path qw(make_path);
> +use File::Basename qw(dirname);
> +use IO::File;
> +use POSIX;
> +
> +use PVE::ProcFSTools;
> +use PVE::Tools qw(
> + file_copy
> + file_get_contents
> + file_set_contents
> + run_command
> +);
> +
> +use base qw(PVE::Storage::Plugin);
> +
> +my $CLUSTER_KNOWN_HOSTS = "/etc/pve/priv/known_hosts";
> +
> +# Plugin Definition
> +
> +sub api {
> + return 11;
> +}
> +
> +sub type {
> + return 'sshfs';
> +}
> +
> +sub plugindata {
> + return {
> + content => [
> + {
> + images => 1,
> + rootdir => 1,
> + vztmpl => 1,
> + iso => 1,
> + backup => 1,
> + snippets => 1,
> + none => 1,
Argh, there's one thing I just noticed: The `none` content type above
should not be declared here and needs to be removed, because it's for
internal use only (for now, at least).
Even though we'll ship this example ourselves, it should of course still
adhere to what third-party plugin devs should do. >.>
For now it doesn't hurt if it's here and can just be removed in a
follow-up, but I'll already intentionally exclude it from the upcoming
plugin dev guide.
> + },
> + {
> + images => 1,
> + rootdir => 1,
> + },
> + ],
> + format => [
> + {
> + raw => 1,
> + qcow2 => 1,
> + vmdk => 1,
> + },
> + 'qcow2',
> + ],
> + 'sensitive-properties' => {
> + 'sshfs-private-key' => 1,
> + },
> + };
> +}
> +
> +sub properties {
> + return {
> + 'sshfs-remote-path' => {
> + description => "Path on the remote filesystem used for SSHFS. Must be absolute.",
> + type => 'string',
> + format => 'pve-storage-path',
> + },
> + 'sshfs-private-key' => {
> + description => "Path to the private key to use for SSHFS.",
> + type => 'string',
> + format => 'pve-storage-path',
> + }
> + };
> +}
> +
> +sub options {
> + return {
> + disable => { optional => 1 },
> + path => { fixed => 1 },
> + 'create-base-path' => { optional => 1 },
> + content => { optional => 1 },
> + 'create-subdirs' => { optional => 1 },
> + 'content-dirs' => { optional => 1 },
> + 'prune-backups' => { optional => 1 },
> + 'max-protected-backups' => { optional => 1 },
> + format => { optional => 1 },
> + bwlimit => { optional => 1 },
> + preallocation => { optional => 1 },
> + nodes => { optional => 1 },
> + shared => { optional => 1 },
> +
> + # SSHFS Options
> + username => {},
> + server => {},
> + 'sshfs-remote-path' => {},
> + port => { optional => 1 },
> + 'sshfs-private-key' => { optional => 1 },
> + };
> +}
> +
> +# SSHFS Helpers
> +
> +my sub sshfs_remote_from_config: prototype($) ($scfg) {
> + my ($user, $host, $remote_path) = $scfg->@{qw(username server sshfs-remote-path)};
> + return "${user}\@${host}:${remote_path}";
> +}
> +
> +my sub sshfs_private_key_path: prototype($) ($storeid) {
> + return "/etc/pve/priv/storage/$storeid.key";
> +}
> +
> +my sub sshfs_common_ssh_opts: prototype($) ($storeid) {
> + my $private_key_path = sshfs_private_key_path($storeid);
> +
> + my @common_opts = (
> + '-o', "UserKnownHostsFile=${CLUSTER_KNOWN_HOSTS}",
> + '-o', 'GlobalKnownHostsFile=none',
> + '-o', "IdentityFile=${private_key_path}",
> + );
> +
> + return @common_opts;
> +}
> +
> +my sub sshfs_set_private_key: prototype($$) ($storeid, $src_key_path) {
> + die "path of private key file not specified" if !defined($src_key_path);
> + die "path of private key file does not exist" if ! -e $src_key_path;
> + die "path of private key file does not point to a file" if ! -f $src_key_path;
> +
> + my $dest_key_path = sshfs_private_key_path($storeid);
> +
> + my $dest_key_parent_dir = dirname($dest_key_path);
> + if (! -e $dest_key_parent_dir) {
> + make_path($dest_key_parent_dir, { chmod => 0700 });
> + } else {
> + die "'$dest_key_path' already exists" if -e $dest_key_path;
> + }
> +
> + file_copy($src_key_path, $dest_key_path, undef, 600);
> +
> + return undef;
> +}
> +
> +my sub sshfs_remove_private_key: prototype($) ($storeid) {
> + my $key_path = sshfs_private_key_path($storeid);
> + unlink($key_path) or $! == ENOENT or die "failed to remove private key '$key_path' - $!\n";
> +
> + return undef;
> +}
> +
> +my sub sshfs_is_mounted: prototype($) ($scfg) {
> + my $remote = sshfs_remote_from_config($scfg);
> +
> + my $mountpoint = Cwd::realpath($scfg->{path}); # Resolve symlinks
> + return 0 if !defined($mountpoint);
> +
> + my $mountdata = PVE::ProcFSTools::parse_proc_mounts();
> +
> + my $has_found_mountpoint = grep {
> + $_->[0] =~ m|^\Q${remote}\E$|
> + && $_->[1] eq $mountpoint
> + && $_->[2] eq 'fuse.sshfs'
> + } $mountdata->@*;
> +
> + return $has_found_mountpoint != 0;
> +}
> +
> +my sub sshfs_mount: prototype($$) ($scfg, $storeid) {
> + my $remote = sshfs_remote_from_config($scfg);
> + my ($port, $mountpoint) = $scfg->@{qw(port path)};
> +
> + my @common_opts = sshfs_common_ssh_opts($storeid);
> + my $cmd = [
> + '/usr/bin/sshfs', @common_opts,
> + '-o', 'noatime',
> + ];
> +
> + push($cmd->@*, '-p', $port) if $port;
> + push($cmd->@*, $remote, $mountpoint);
> +
> + eval {
> + run_command(
> + $cmd,
> + timeout => 10,
> + errfunc => sub { warn "$_[0]\n"; },
> + );
> + };
> + if (my $err = $@) {
> + die "failed to mount SSHFS storage '$remote' at '$mountpoint': $@\n";
> + }
> +
> + die "SSHFS storage '$remote' not mounted at '$mountpoint' despite reported success\n"
> + if ! sshfs_is_mounted($scfg);
> +
> + return;
> +}
> +
> +my sub sshfs_umount: prototype($) ($scfg) {
> + my $mountpoint = $scfg->{path};
> +
> + my $cmd = ['/usr/bin/umount', $mountpoint];
> +
> + eval {
> + run_command(
> + $cmd,
> + timeout => 10,
> + errfunc => sub { warn "$_[0]\n"; },
> + );
> + };
> + if (my $err = $@) {
> + die "failed to unmount SSHFS at '$mountpoint': $err\n";
> + }
> +
> + return;
> +}
> +
> +# Storage Implementation
> +
> +sub on_add_hook ($class, $storeid, $scfg, %sensitive) {
> + $scfg->{shared} = 1; # mark SSHFS storages as shared by default
> +
> + eval {
> + my $src_key_path = $sensitive{'sshfs-private-key'};
> + sshfs_set_private_key($storeid, $src_key_path);
> + };
> + die "error while adding SSHFS storage '${storeid}': $@\n" if $@;
> +
> + return undef;
> +}
> +
> +sub on_update_hook ($class, $storeid, $scfg, %sensitive) {
> + return undef if !exists($sensitive{'sshfs-private-key'});
> +
> + my $src_key_path = $sensitive{'sshfs-private-key'};
> +
> + if (!defined($src_key_path)) {
> + warn "removing private key for SSHFS storage '${storeid}'";
> + warn "the storage might not be mountable without a private key!";
> +
> + eval { sshfs_remove_private_key($storeid); };
> + die $@ if $@;
> +
> + return undef;
> + }
> +
> + my $dest_key_path = sshfs_private_key_path($storeid);
> + my $dest_key_path_tmp = "${dest_key_path}.old";
> +
> + file_copy($dest_key_path, $dest_key_path_tmp) if -e $dest_key_path;
> +
> + eval { file_copy($src_key_path, $dest_key_path, undef, 600); };
> +
> + if (my $err = $@) {
> + if (-e $dest_key_path_tmp) {
> + warn "attempting to restore previous private key for storage '${storeid}'\n";
> + eval { file_copy($dest_key_path_tmp, $dest_key_path, undef, 600); };
> + warn "$@\n" if $@;
> +
> + unlink $dest_key_path_tmp;
> + }
> +
> + die "failed to set private key for SSHFS storage '${storeid}': $err\n";
> + }
> +
> + unlink $dest_key_path_tmp;
> +
> + return undef;
> +}
> +
> +sub on_delete_hook ($class, $storeid, $scfg) {
> + eval { sshfs_remove_private_key($storeid); };
> + warn $@ if $@;
> +
> + eval { sshfs_umount($scfg) if sshfs_is_mounted($scfg); };
> + warn $@ if $@;
> +
> + return undef;
> +}
> +
> +sub check_connection ($class, $storeid, $scfg) {
> + my ($user, $host, $port) = $scfg->@{qw(username server port)};
> +
> + my @common_opts = sshfs_common_ssh_opts($storeid);
> + my $cmd = [
> + '/usr/bin/ssh',
> + '-T',
> + @common_opts,
> + '-o', 'BatchMode=yes',
> + '-o', 'ConnectTimeout=5',
> + ];
> +
> + push($cmd->@*, "-p", $port) if $port;
> + push($cmd->@*, "${user}\@${host}", 'exit 0');
> +
> + eval {
> + run_command(
> + $cmd,
> + timeout => 10,
> + errfunc => sub { warn "$_[0]\n"; },
> + );
> + };
> + if (my $err = $@) {
> + warn "$err";
> + return 0;
> + }
> +
> + return 1;
> +}
> +
> +sub activate_storage ($class, $storeid, $scfg, $cache) {
> + my $mountpoint = $scfg->{path};
> +
> + if (!sshfs_is_mounted($scfg)) {
> + if ($scfg->{'create-base-path'} // 1) {
> + make_path($mountpoint);
> + }
> +
> + die "unable to activate storage '$storeid' - directory '$mountpoint' does not exist\n"
> + if ! -d $mountpoint;
> +
> + sshfs_mount($scfg, $storeid);
> + }
> +
> + $class->SUPER::activate_storage($storeid, $scfg, $cache);
> + return;
> +}
> +
> +sub get_volume_attribute ($class, $scfg, $storeid, $volname, $attribute) {
> + my ($vtype) = $class->parse_volname($volname);
> + return if $vtype ne 'backup';
> +
> + my $volume_path = PVE::Storage::Plugin->filesystem_path($scfg, $volname);
> +
> + if ($attribute eq 'notes') {
> + my $notes_path = $volume_path . $class->SUPER::NOTES_EXT;
> +
> + if (-f $notes_path) {
> + my $notes = file_get_contents($notes_path);
> + return eval { decode('UTF-8', $notes, 1) } // $notes;
> + }
> +
> + return "";
> + }
> +
> + if ($attribute eq 'protected') {
> + return -e PVE::Storage::protection_file_path($volume_path) ? 1 : 0;
> + }
> +
> + return;
> +}
> +
> +sub update_volume_attribute ($class, $scfg, $storeid, $volname, $attribute, $value) {
> + my ($vtype, $name) = $class->parse_volname($volname);
> + die "only backups support attribute '$attribute'\n" if $vtype ne 'backup';
> +
> + my $volume_path = PVE::Storage::Plugin->filesystem_path($scfg, $volname);
> +
> + if ($attribute eq 'notes') {
> + my $notes_path = $volume_path . $class->SUPER::NOTES_EXT;
> +
> + if (defined($value)) {
> + my $encoded_notes = encode('UTF-8', $value);
> + file_set_contents($notes_path, $encoded_notes);
> + } else {
> + unlink $notes_path or $! == ENOENT or die "could not delete notes - $!\n";
> + }
> +
> + return;
> + }
> +
> + if ($attribute eq 'protected') {
> + my $protection_path = PVE::Storage::protection_file_path($volume_path);
> +
> + # Protection already set or unset
> + return if !((-e $protection_path) xor $value);
> +
> + if ($value) {
> + my $fh = IO::File->new($protection_path, O_CREAT, 0644)
> + or die "unable to create protection file '$protection_path' - $!\n";
> + close($fh);
> + } else {
> + unlink $protection_path or $! == ENOENT
> + or die "could not delete protection file '$protection_path' - $!\n";
> + }
> +
> + return;
> + }
> +
> + die "attribute '$attribute' is not supported for storage type '$scfg->{type}'\n";
> +}
> +
> +1;
More information about the pve-devel
mailing list