[pve-devel] [PATCH ha-manager 1/4] add possibility to simulate locks from services

[Thomas Lamprecht]

In the real PVE2 environment locks can be leftover if a node fails
during an locked action like for example backups.
A left over lock may hinder the service to start on another node,
allow our test environment to simulate such events.

Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
 src/PVE/HA/Sim/Hardware.pm                    | 38 +++++++++++++++++++++++++++
 src/PVE/HA/Sim/Resources.pm                   |  5 ++++
 src/PVE/HA/Sim/TestHardware.pm                |  9 +++++++
 src/test/test-locked-service1/README          |  3 +++
 src/test/test-locked-service1/cmdlist         |  5 ++++
 src/test/test-locked-service1/hardware_status |  5 ++++
 src/test/test-locked-service1/manager_status  |  1 +
 src/test/test-locked-service1/service_config  |  3 +++
 8 files changed, 69 insertions(+)
 create mode 100644 src/test/test-locked-service1/README
 create mode 100644 src/test/test-locked-service1/cmdlist
 create mode 100644 src/test/test-locked-service1/hardware_status
 create mode 100644 src/test/test-locked-service1/manager_status
 create mode 100644 src/test/test-locked-service1/service_config

diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index be1037d..8a7734c 100644
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -194,6 +194,44 @@ sub change_service_location {
     $self->write_service_config($conf);
 }
 
+sub service_has_lock {
+    my ($self, $sid) = @_;
+
+    my $conf = $self->read_service_config();
+
+    die "no such service '$sid'\n" if !$conf->{$sid};
+
+    return $conf->{$sid}->{lock};
+}
+
+sub lock_service {
+    my ($self, $sid) = @_;
+
+    my $conf = $self->read_service_config();
+
+    die "no such service '$sid'\n" if !$conf->{$sid};
+
+    $conf->{$sid}->{lock} = 'virtual';
+
+    $self->write_service_config($conf);
+
+    return $conf;
+}
+
+sub unlock_service {
+    my ($self, $sid) = @_;
+
+    my $conf = $self->read_service_config();
+
+    die "no such service '$sid'\n" if !$conf->{$sid};
+
+    my $removed_lock = delete $conf->{$sid}->{lock};
+
+    $self->write_service_config($conf);
+
+    return $removed_lock;
+}
+
 sub queue_crm_commands_nolock {
     my ($self, $cmd) = @_;
 
diff --git a/src/PVE/HA/Sim/Resources.pm b/src/PVE/HA/Sim/Resources.pm
index 25b034e..ec3d775 100644
--- a/src/PVE/HA/Sim/Resources.pm
+++ b/src/PVE/HA/Sim/Resources.pm
@@ -42,6 +42,11 @@ sub start {
     my $hardware = $haenv->hardware();
     my $ss = $hardware->read_service_status($nodename);
 
+    if ($hardware->service_has_lock("$service_type:$id")) {
+	$haenv->log('err', "service '$service_type:$id' locked, unable to start!");
+	return;
+    }
+
     $haenv->sleep(2);
 
     $ss->{"$service_type:$id"} = 1;
diff --git a/src/PVE/HA/Sim/TestHardware.pm b/src/PVE/HA/Sim/TestHardware.pm
index cfd48e7..45c3102 100644
--- a/src/PVE/HA/Sim/TestHardware.pm
+++ b/src/PVE/HA/Sim/TestHardware.pm
@@ -89,6 +89,7 @@ sub log {
 # restart-lrm <node>
 # service <sid> <enabled|disabled>
 # service <sid> <migrate|relocate> <target>
+# service <sid> lock/unlock
 
 sub sim_hardware_cmd {
     my ($self, $cmdstr, $logid) = @_;
@@ -193,6 +194,14 @@ sub sim_hardware_cmd {
 
 		$self->delete_service($sid);
 
+	    } elsif ($action eq 'lock') {
+
+		$self->lock_service($sid);
+
+	    } elsif ($action eq 'unlock') {
+
+		$self->unlock_service($sid);
+
 	    } else {
 		die "sim_hardware_cmd: unknown service action '$action' " .
 		    "- not implemented\n"
diff --git a/src/test/test-locked-service1/README b/src/test/test-locked-service1/README
new file mode 100644
index 0000000..1c4f7ed
--- /dev/null
+++ b/src/test/test-locked-service1/README
@@ -0,0 +1,3 @@
+Test failover after single node network failure and a interrupted backup,
+simulated by a left over lock. The lock should be removed on service recovery as
+the cause it protected vanished after the fencing of node3.
diff --git a/src/test/test-locked-service1/cmdlist b/src/test/test-locked-service1/cmdlist
new file mode 100644
index 0000000..9bdf6bb
--- /dev/null
+++ b/src/test/test-locked-service1/cmdlist
@@ -0,0 +1,5 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on"],
+    [ "service vm:103 lock" ],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-locked-service1/hardware_status b/src/test/test-locked-service1/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-locked-service1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-locked-service1/manager_status b/src/test/test-locked-service1/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-locked-service1/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-locked-service1/service_config b/src/test/test-locked-service1/service_config
new file mode 100644
index 0000000..c6860e7
--- /dev/null
+++ b/src/test/test-locked-service1/service_config
@@ -0,0 +1,3 @@
+{
+    "vm:103": { "node": "node3", "state": "enabled" }
+}
-- 
2.1.4