[pve-devel] [PATCH qemu-server 1/4] mtunnel: add API endpoints
Fabian Grünbichler
f.gruenbichler at proxmox.com
Fri Mar 6 11:20:32 CET 2020
the following three endpoints are used for migration on the remote side
GET /nodes/NODE/qemu/VMID/mtunnel
returns identifier for this migration and ticket.
both are passed to the other two endpoints to correspond that calls
belong to the same migration run, and that permissions have been
properly validated.
note: neither ticket creation nor validation nor permission checks are
implemented yet ;)
POST /nodes/NODE/qemu/VMID/mtunnel
which creates and locks an empty VM config, and spawns the main qmtunnel
worker which binds to a VM-specific UNIX socket.
this worker handles migration commands coming in via this UNIX socket:
- config <BASE64(JSON($config))>
- start (returning migration info)
- resume
- stop
- unlock
- cleanup (not yet implemented)
this worker serves as a replacement for both 'qm mtunnel' and various
manual calls via SSH.
GET+WebSocket upgrade /nodes/NODE/qemu/VMID/mtunnelwebsocket
gets called for connecting to a UNIX socket or TCP port via websocket
forwarding, i.e. once for the main command mtunnel, and once each for
the memory migration and each NBD drive-mirror.
Signed-off-by: Fabian Grünbichler <f.gruenbichler at proxmox.com>
---
PVE/API2/Qemu.pm | 292 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 292 insertions(+)
diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index caca430..24f0dfd 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -6,6 +6,9 @@ use Cwd 'abs_path';
use Net::SSLeay;
use POSIX;
use IO::Socket::IP;
+use IO::Socket::UNIX;
+use JSON;
+use MIME::Base64;
use URI::Escape;
use PVE::Cluster qw (cfs_read_file cfs_write_file);;
@@ -715,6 +718,7 @@ __PACKAGE__->register_method({
{ subdir => 'spiceproxy' },
{ subdir => 'sendkey' },
{ subdir => 'firewall' },
+ { subdir => 'mtunnel' },
];
return $res;
@@ -4070,4 +4074,292 @@ __PACKAGE__->register_method({
return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
}});
+# TODO add get call to retrieve tunnel ticket
+
+
+# TODO verify ticket in mtunnel POST
+
+__PACKAGE__->register_method({
+ name => 'mtunnel',
+ path => '{vmid}/mtunnel',
+ method => 'POST',
+ protected => 1,
+ proxyto => 'node',
+ description => 'Migration tunnel endpoint - only for internal use by VM migration.',
+ permissions => {
+ description => "You need 'VM.Allocate' permissions on /vms/{vmid}, as well as 'Datastore.AllocateSpace' on any used storage.",
+ user => 'all', # check inside
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ version => {
+ type => 'integer',
+ optional => 0,
+ },
+ vmid => get_standard_option('pve-vmid'),
+ },
+ },
+ returns => {
+ type => 'string',
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $node = extract_param($param, 'node');
+ my $vmid = extract_param($param, 'vmid');
+
+ my $lock = 'create';
+
+ PVE::Cluster::check_cfs_quorum();
+
+ eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); };
+
+ raise_param_exc({ vmid => "unable to create empty VM config - $@"})
+ if $@;
+
+ my $realcmd = sub {
+ my $tunnel_socket;
+ my $pveproxy_uid;
+
+ my $run_locked = sub {
+ my ($code) = @_;
+ PVE::QemuConfig->lock_config($vmid, sub {
+ my $conf = PVE::QemuConfig->load_config($vmid);
+
+ die "Encountered wrong lock - aborting mtunnel command handling.\n"
+ if !PVE::QemuConfig->has_lock($conf, $lock);
+
+ $code->();
+ });
+ };
+
+ $run_locked->(sub {
+ my $socket_addr = "/run/qemu-server/$vmid.mtunnel";
+ unlink $socket_addr;
+
+ $tunnel_socket = IO::Socket::UNIX->new(
+ Type => SOCK_STREAM(),
+ Local => $socket_addr,
+ Listen => 1,
+ );
+
+ $pveproxy_uid = getpwnam('www-data')
+ or die "Failed to resolve user 'www-data' to numeric UID\n";
+ chown $pveproxy_uid, -1, $socket_addr;
+ });
+
+ print "mtunnel started\n";
+
+ my $conn = $tunnel_socket->accept();
+ $conn->print("tunnel online\n");
+ $conn->print("ver 2\n");
+
+ while (my $line = <$conn>) {
+ chomp $line;
+ print "command received: '$line'\n";
+ if ($line =~ /^config (.*)$/) {
+ my $conf_str = $1;
+ eval {
+ my $newconf = JSON::decode_json(MIME::Base64::decode_base64url($conf_str));
+ $newconf->{lock} = $lock;
+ $run_locked->(sub {
+ PVE::QemuConfig->write_config($vmid, $newconf);
+ });
+ $conn->print("OK\n");
+ };
+
+ $conn->print("ERR: failed to write config - $@\n") if $@;
+ } elsif ($line =~ /^start (.*)$/) {
+ my $param_str = $1;
+ eval {
+ my $param = JSON::decode_json($param_str);
+ print "start params:\n", Dumper($param), "\n";
+ my $raddr;
+ my $rport;
+ my $ruri; # the whole migration dst. URI (protocol:address[:port])
+ my $spice_port;
+ my $drive_info = {};
+ my $nbd;
+ my $log = "";
+ $run_locked->(sub {
+ PVE::Cluster::check_cfs_quorum();
+ my $output;
+ open my $output_fh, '>>', \$output;
+
+ select $output_fh;
+ my $storecfg = PVE::Storage::config();
+ PVE::QemuServer::vm_start($storecfg, $vmid, $param->{state_uri}, 1, $node, undef, $param->{machine}, $param->{spice}, $param->{network}, $param->{migration_type}, $param->{targetstorage});
+ select STDOUT;
+ # TODO weed out legacy stuff
+ foreach my $line (split("\n", $output)) {
+ chomp $line;
+ next if $line eq '';
+
+ if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) {
+ $raddr = $1;
+ $rport = int($2);
+ $ruri = "tcp:$raddr:$rport";
+ }
+ elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) {
+ $raddr = $1;
+ die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
+ chown $pveproxy_uid, -1, $raddr;
+ $ruri = "unix:$raddr";
+ }
+ elsif ($line =~ m/^migration listens on port (\d+)$/) {
+ $raddr = "localhost";
+ $rport = int($1);
+ $ruri = "tcp:$raddr:$rport";
+ }
+ elsif ($line =~ m/^spice listens on port (\d+)$/) {
+ $spice_port = int($1);
+ }
+ elsif ($line =~ m!^storage migration listens on nbd:unix:(/run/qemu-server/(\d+)_nbd\.migrate):exportname=(\S+) volume:(\S+)$!) {
+ die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2;
+ my $nbd_unix_addr = $1;
+ my $nbd_uri = "nbd:unix:$nbd_unix_addr:exportname=$3";
+ my $targetdrive = $3;
+ my $drivestr = $4;
+ $targetdrive =~ s/drive-//g;
+ $log .= "$targetdrive: $drivestr ('$line')\n";
+
+ $drive_info->{$targetdrive}->{drivestr} = $drivestr;
+ $drive_info->{$targetdrive}->{nbd_uri} = $nbd_uri;
+
+ die "Migration returned two different NBD addresses: '$nbd_uri' / '$nbd'\n"
+ if $nbd && $nbd_unix_addr ne $nbd;
+ $nbd = $nbd_unix_addr;
+ chown $pveproxy_uid, -1, $nbd_unix_addr;
+ } else {
+ $log .= "[remote] $line\n";
+ }
+ }
+ });
+
+ die "$@\n" if $@;
+
+ select STDOUT;
+ die "unable to detect remote migration address\n" if !$raddr || !$ruri;
+ my $res = {
+ raddr => $raddr,
+ rport => $rport,
+ ruri => $ruri,
+ drives => $drive_info,
+ nbd => $nbd,
+ };
+ $res->{spice_port} = $spice_port if $spice_port;
+ my $reply = JSON::encode_json($res);
+ $conn->print("OK\n");
+ $conn->print("$reply\n");
+ };
+ $conn->print("ERR: failed to start VM - $@\n") if $@;
+ } elsif ($line =~ /^stop$/) {
+ eval {
+ $run_locked->(sub {
+ PVE::QemuServer::vm_stop(undef, $vmid, 1, 1);
+ });
+ $conn->print("OK\n");
+ };
+ $conn->print("ERR: failed to stop VM - $@\n") if $@;
+ } elsif ($line =~ /^nbdstop$/) {
+ eval {
+ $run_locked->(sub {
+ PVE::QemuServer::nbd_stop($vmid);
+ });
+ $conn->print("OK\n");
+ };
+ $conn->print("ERR: failed to stop NBD server - $@\n") if $@;
+ } elsif ($line =~ /^resume /) {
+ eval {
+ $run_locked->(sub {
+ if (PVE::QemuServer::check_running($vmid, 1)) {
+ PVE::QemuServer::vm_resume($vmid, 1, 1);
+ } else {
+ die "VM $vmid not running\n";
+ }
+ });
+ $conn->print("OK\n");
+ };
+ $conn->print("ERR: resume failed - $@") if $@;
+ } elsif ($line =~ /^unlock$/) {
+ eval {
+ PVE::QemuConfig->remove_lock($vmid, $lock);
+ $conn->print("OK\n");
+ };
+ $conn->print("ERR: unlock failed - $@") if $@;
+ } elsif ($line =~ /^quit$/) {
+ $conn->print("OK\n");
+ $conn->close();
+ $tunnel_socket->close();
+ last;
+ } else {
+ $conn->print("ERR: unknown command '$line'");
+ }
+ }
+
+ print "mtunnel exited\n";
+ };
+
+ $rpcenv->fork_worker('qmtunnel', $vmid, $authuser, $realcmd);
+ }});
+
+__PACKAGE__->register_method({
+ name => 'mtunnelwebsocket',
+ path => '{vmid}/mtunnelwebsocket',
+ method => 'GET',
+ proxyto => 'node',
+ permissions => {
+ description => "You need to pass a ticket valid for the selected port/socket. Tickets can be created via the mtunnel API call, which will check permissions accordingly.",
+ user => 'all', # check inside
+ },
+ description => 'Migration tunnel endpoint for websocket upgrade - only for internal use by VM migration.',
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ vmid => get_standard_option('pve-vmid'),
+ port => {
+ type => "integer",
+ description => "TCP port to forward to",
+ optional => 1,
+ },
+ socket => {
+ type => "string",
+ description => "unix socket to forward to",
+ optional => 1,
+ },
+ },
+ },
+ returns => {
+ type => "object",
+ properties => {
+ port => { type => 'string', optional => 1 },
+ socket => { type => 'string', optional => 1 },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $authuser = $rpcenv->get_user();
+
+ my $vmid = $param->{vmid};
+ my $node = $param->{node};
+
+ # TODO: verify ticket here
+
+ my $port = $param->{port};
+ my $socket = $param->{socket};
+
+ # TODO either or
+
+ return { port => $port } if $port;
+ return { socket => $socket } if $socket;
+ }});
+
1;
--
2.20.1
More information about the pve-devel
mailing list