itest: add itest for failover after forcefull shutdown

This commit is contained in:
Andras Banki-Horvath 2021-03-11 17:23:18 +01:00
parent 5d8488871c
commit 7caf26ce94
No known key found for this signature in database
GPG Key ID: 80E5375C094198D8
3 changed files with 61 additions and 3 deletions

@ -788,6 +788,11 @@ func (n *NetworkHarness) ShutdownNode(node *HarnessNode) error {
return nil return nil
} }
// KillNode kills the node (but won't wait for the node process to stop).
func (n *NetworkHarness) KillNode(node *HarnessNode) error {
return node.kill()
}
// StopNode stops the target node, but doesn't yet clean up its directories. // StopNode stops the target node, but doesn't yet clean up its directories.
// This can be used to temporarily bring a node down during a test, to be later // This can be used to temporarily bring a node down during a test, to be later
// started up again. // started up again.

@ -5,6 +5,7 @@ package itest
import ( import (
"context" "context"
"io/ioutil" "io/ioutil"
"testing"
"time" "time"
"github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil"
@ -29,7 +30,41 @@ func assertLeader(ht *harnessTest, observer cluster.LeaderElector,
} }
} }
// testEtcdFailover tests that in a cluster setup where two LND nodes form a
// single cluster (sharing the same identity) one can hand over the leader role
// to the other (failing over after graceful shutdown or forceful abort).
func testEtcdFailover(net *lntest.NetworkHarness, ht *harnessTest) { func testEtcdFailover(net *lntest.NetworkHarness, ht *harnessTest) {
testCases := []struct {
name string
kill bool
}{{
name: "failover after shutdown",
kill: false,
}, {
name: "failover after abort",
kill: true,
}}
for _, test := range testCases {
test := test
ht.t.Run(test.name, func(t1 *testing.T) {
ht1 := newHarnessTest(t1, ht.lndHarness)
ht1.RunTestCase(&testCase{
name: test.name,
test: func(_ *lntest.NetworkHarness,
tt *harnessTest) {
testEtcdFailoverCase(net, tt, test.kill)
},
})
})
}
}
func testEtcdFailoverCase(net *lntest.NetworkHarness, ht *harnessTest,
kill bool) {
ctxb := context.Background() ctxb := context.Background()
tmpDir, err := ioutil.TempDir("", "etcd") tmpDir, err := ioutil.TempDir("", "etcd")
@ -104,9 +139,22 @@ func testEtcdFailover(net *lntest.NetworkHarness, ht *harnessTest) {
FeeLimitSat: noFeeLimitMsat, FeeLimitSat: noFeeLimitMsat,
}) })
// Shut down Carol-1 and wait for Carol-2 to become the leader. // Shut down or kill Carol-1 and wait for Carol-2 to become the leader.
shutdownAndAssert(net, ht, carol1) var failoverTimeout time.Duration
err = carol2.WaitUntilLeader(30 * time.Second) if kill {
err = net.KillNode(carol1)
if err != nil {
ht.Fatalf("Can't kill Carol-1: %v", err)
}
failoverTimeout = 2 * time.Minute
} else {
shutdownAndAssert(net, ht, carol1)
failoverTimeout = 30 * time.Second
}
err = carol2.WaitUntilLeader(failoverTimeout)
if err != nil { if err != nil {
ht.Fatalf("Waiting for Carol-2 to become the leader failed: %v", ht.Fatalf("Waiting for Carol-2 to become the leader failed: %v",
err) err)

@ -1147,6 +1147,11 @@ func (hn *HarnessNode) shutdown() error {
return nil return nil
} }
// kill kills the lnd process
func (hn *HarnessNode) kill() error {
return hn.cmd.Process.Kill()
}
// closeChanWatchRequest is a request to the lightningNetworkWatcher to be // closeChanWatchRequest is a request to the lightningNetworkWatcher to be
// notified once it's detected within the test Lightning Network, that a // notified once it's detected within the test Lightning Network, that a
// channel has either been added or closed. // channel has either been added or closed.