[pve-devel] [PATCH http-server v2 2/5] accept-phase: fix conn_count "leak"

[Stoiko Ivanov]

When handling new connections in 'accept_connections' the number of
active connections got increased before the AnyEvent::Handle
registered the callback which would decrement it on error/eof.

Any error/die beforehand would skip the decrement, and leave the
process in an endless loop upon exiting in wait_end_loop.

This can happen e.g. when the call to getpeername fails, or if the
connection is denied by the ALLOW_FROM/DENY_FROM settings in
'/etc/default/pveproxy' (which is also a simple reproducer for
that).

Additionally it can cause a denial of service, by attempting to
connect from a denied ip until the connection count exeeds the maximum
connections of all child-processes.

Should the connection count become negative a warning is logged in both
places where we decrement it, in case of a failed AnyEvent::Handle->new(),
the count is not decremented if this would happen.

Reported via our community-forum:
https://forum.proxmox.com/threads/pveproxy-eats-available-ram.79617/

Co-Authored-by: Dominik Csapak <d.csapak at proxmox.com>
Signed-off-by: Stoiko Ivanov <s.ivanov at proxmox.com>
---
 PVE/APIServer/AnyEvent.pm | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/PVE/APIServer/AnyEvent.pm b/PVE/APIServer/AnyEvent.pm
index 7916bdd..af2fde8 100644
--- a/PVE/APIServer/AnyEvent.pm
+++ b/PVE/APIServer/AnyEvent.pm
@@ -157,6 +157,11 @@ sub client_do_disconnect {
 
     &$shutdown_hdl($hdl);
 
+    if ($self->{conn_count} <= 0) {
+	my $msg = "connection count <= 0!\n";
+	syslog('warn', $msg);
+	$self->dprint("warn: $msg");
+    }
     $self->{conn_count}--;
 
     print "$$: CLOSE FH" .  $hdl->{fh}->fileno() . " CONN$self->{conn_count}\n" if $self->{debug};
@@ -1489,8 +1494,6 @@ sub accept {
 
     fh_nonblocking $clientfh, 1;
 
-    $self->{conn_count}++;
-
     return $clientfh;
 }
 
@@ -1564,6 +1567,7 @@ sub check_host_access {
 sub accept_connections {
     my ($self) = @_;
 
+    my $hdl_err;
     eval {
 
 	while (my $clientfh = $self->accept()) {
@@ -1571,7 +1575,7 @@ sub accept_connections {
 	    my $reqstate = { keep_alive => $self->{keep_alive} };
 
 	    # stop keep-alive when there are many open connections
-	    if ($self->{conn_count} >= $self->{max_conn_soft_limit}) {
+	    if ($self->{conn_count} + 1 >= $self->{max_conn_soft_limit}) {
 		$reqstate->{keep_alive} = 0;
 	    }
 
@@ -1587,6 +1591,8 @@ sub accept_connections {
 		next;
 	    }
 
+	    $hdl_err = 1;
+	    $self->{conn_count}++;
 	    $reqstate->{hdl} = AnyEvent::Handle->new(
 		fh => $clientfh,
 		rbuf_max => 64*1024,
@@ -1609,6 +1615,7 @@ sub accept_connections {
 		    if (my $err = $@) { syslog('err', "$err"); }
 		},
 		($self->{tls_ctx} ? (tls => "accept", tls_ctx => $self->{tls_ctx}) : ()));
+	    $hdl_err = 0;
 
 	    print "$$: ACCEPT FH" .  $clientfh->fileno() . " CONN$self->{conn_count}\n" if $self->{debug};
 
@@ -1618,6 +1625,15 @@ sub accept_connections {
 
     if (my $err = $@) {
 	syslog('err', $err);
+	if ($hdl_err) {
+	    if ($self->{conn_count} <= 0) {
+		my $msg = "connection count <= 0 not decrementing!\n";
+		syslog('warn', $msg);
+		$self->dprint("warn: $msg");
+	    } else {
+		$self->{conn_count}--;
+	    }
+	}
 	$self->{end_loop} = 1;
     }
 
-- 
2.20.1