itest: add itest for failover after forcefull shutdown
This commit is contained in:
parent
5d8488871c
commit
7caf26ce94
@ -788,6 +788,11 @@ func (n *NetworkHarness) ShutdownNode(node *HarnessNode) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// KillNode kills the node (but won't wait for the node process to stop).
|
||||||
|
func (n *NetworkHarness) KillNode(node *HarnessNode) error {
|
||||||
|
return node.kill()
|
||||||
|
}
|
||||||
|
|
||||||
// StopNode stops the target node, but doesn't yet clean up its directories.
|
// StopNode stops the target node, but doesn't yet clean up its directories.
|
||||||
// This can be used to temporarily bring a node down during a test, to be later
|
// This can be used to temporarily bring a node down during a test, to be later
|
||||||
// started up again.
|
// started up again.
|
||||||
|
@ -5,6 +5,7 @@ package itest
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/btcsuite/btcutil"
|
"github.com/btcsuite/btcutil"
|
||||||
@ -29,7 +30,41 @@ func assertLeader(ht *harnessTest, observer cluster.LeaderElector,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// testEtcdFailover tests that in a cluster setup where two LND nodes form a
|
||||||
|
// single cluster (sharing the same identity) one can hand over the leader role
|
||||||
|
// to the other (failing over after graceful shutdown or forceful abort).
|
||||||
func testEtcdFailover(net *lntest.NetworkHarness, ht *harnessTest) {
|
func testEtcdFailover(net *lntest.NetworkHarness, ht *harnessTest) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
kill bool
|
||||||
|
}{{
|
||||||
|
name: "failover after shutdown",
|
||||||
|
kill: false,
|
||||||
|
}, {
|
||||||
|
name: "failover after abort",
|
||||||
|
kill: true,
|
||||||
|
}}
|
||||||
|
|
||||||
|
for _, test := range testCases {
|
||||||
|
test := test
|
||||||
|
|
||||||
|
ht.t.Run(test.name, func(t1 *testing.T) {
|
||||||
|
ht1 := newHarnessTest(t1, ht.lndHarness)
|
||||||
|
ht1.RunTestCase(&testCase{
|
||||||
|
name: test.name,
|
||||||
|
test: func(_ *lntest.NetworkHarness,
|
||||||
|
tt *harnessTest) {
|
||||||
|
|
||||||
|
testEtcdFailoverCase(net, tt, test.kill)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func testEtcdFailoverCase(net *lntest.NetworkHarness, ht *harnessTest,
|
||||||
|
kill bool) {
|
||||||
|
|
||||||
ctxb := context.Background()
|
ctxb := context.Background()
|
||||||
|
|
||||||
tmpDir, err := ioutil.TempDir("", "etcd")
|
tmpDir, err := ioutil.TempDir("", "etcd")
|
||||||
@ -104,9 +139,22 @@ func testEtcdFailover(net *lntest.NetworkHarness, ht *harnessTest) {
|
|||||||
FeeLimitSat: noFeeLimitMsat,
|
FeeLimitSat: noFeeLimitMsat,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Shut down Carol-1 and wait for Carol-2 to become the leader.
|
// Shut down or kill Carol-1 and wait for Carol-2 to become the leader.
|
||||||
shutdownAndAssert(net, ht, carol1)
|
var failoverTimeout time.Duration
|
||||||
err = carol2.WaitUntilLeader(30 * time.Second)
|
if kill {
|
||||||
|
err = net.KillNode(carol1)
|
||||||
|
if err != nil {
|
||||||
|
ht.Fatalf("Can't kill Carol-1: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
failoverTimeout = 2 * time.Minute
|
||||||
|
|
||||||
|
} else {
|
||||||
|
shutdownAndAssert(net, ht, carol1)
|
||||||
|
failoverTimeout = 30 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
err = carol2.WaitUntilLeader(failoverTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ht.Fatalf("Waiting for Carol-2 to become the leader failed: %v",
|
ht.Fatalf("Waiting for Carol-2 to become the leader failed: %v",
|
||||||
err)
|
err)
|
||||||
|
@ -1147,6 +1147,11 @@ func (hn *HarnessNode) shutdown() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// kill kills the lnd process
|
||||||
|
func (hn *HarnessNode) kill() error {
|
||||||
|
return hn.cmd.Process.Kill()
|
||||||
|
}
|
||||||
|
|
||||||
// closeChanWatchRequest is a request to the lightningNetworkWatcher to be
|
// closeChanWatchRequest is a request to the lightningNetworkWatcher to be
|
||||||
// notified once it's detected within the test Lightning Network, that a
|
// notified once it's detected within the test Lightning Network, that a
|
||||||
// channel has either been added or closed.
|
// channel has either been added or closed.
|
||||||
|
Loading…
Reference in New Issue
Block a user