test: add new series of itests for various SCB restore scenarios

In this commit, we add 4 new itests for exercising the SCB restore
process via 4 primary scenarios: recover from backup using RPC, recover
from file using RPC, recover channels during init/creation, recover
channels during unlock. With all fields populated there're a total of 24
new scenarios to cover. At the time of authoring of this commit, the
other scenarios (bits are: initiator, updates, private) have been left
out for now, as they increased the run time of the integration tests
significantly.
This commit is contained in:
Olaoluwa Osuntokun 2019-03-11 17:41:01 -07:00
parent e3029dee45
commit f2160273b7
No known key found for this signature in database
GPG Key ID: CE58F7F8E20FD9A2

@ -13190,7 +13190,7 @@ func testChannelBackupUpdates(net *lntest.NetworkHarness, t *harnessTest) {
chanPoints = append(chanPoints, chanPoint)
}
// Using this helper function, we'll maintain a pointe rot he latest
// Using this helper function, we'll maintain a pointer to the latest
// channel backup so we can compare it to the on disk state.
var currentBackup *lnrpc.ChanBackupSnapshot
assertBackupNtfns := func(numNtfns int) {
@ -13333,7 +13333,10 @@ func testExportChannelBackup(net *lntest.NetworkHarness, t *harnessTest) {
// backups of each of the channels.
for _, chanPoint := range chanPoints {
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
chanBackup, err := carol.ExportChannelBackup(ctxt, chanPoint)
req := &lnrpc.ExportChannelBackupRequest{
ChanPoint: chanPoint,
}
chanBackup, err := carol.ExportChannelBackup(ctxt, req)
if err != nil {
t.Fatalf("unable to fetch backup for channel %v: %v",
chanPoint, err)
@ -13444,6 +13447,401 @@ func testExportChannelBackup(net *lntest.NetworkHarness, t *harnessTest) {
assertNumSingleBackups(0)
assertMultiBackupFound()(false, nil)
}
// nodeRestorer is a function closure that allows each chanRestoreTestCase to
// control exactly *how* the prior node is restored. This might be using an
// backup obtained over RPC, or the file system, etc.
type nodeRestorer func() (*lntest.HarnessNode, error)
// chanRestoreTestCase describes a test case for an end to end SCB restoration
// work flow. One node will start from scratch using an existing SCB. At the
// end of the est, both nodes should be made whole via the DLP protocol.
type chanRestoreTestCase struct {
// name is the name of the target test case.
name string
// channelsUpdated is false then this means that no updates
// have taken place within the channel before restore.
// Otherwise, HTLCs will be settled between the two parties
// before restoration modifying the balance beyond the initial
// allocation.
channelsUpdated bool
// initiator signals if Dave should be the one that opens the
// channel to Alice, or if it should be the other way around.
initiator bool
// private signals if the channel from Dave to Carol should be
// private or not.
private bool
// restoreMethod takes an old node, then returns a function
// closure that'll return the same node, but with its state
// restored via a custom method. We use this to abstract away
// _how_ a node is restored from our assertions once the node
// has been fully restored itself.
restoreMethod func(oldNode *lntest.HarnessNode,
backupFilePath string,
mnemonic []string) (nodeRestorer, error)
}
// testChanRestoreScenario executes a chanRestoreTestCase from end to end,
// ensuring that after Dave restores his channel state according to the
// testCase, the DLP protocol is executed properly and both nodes are made
// whole.
func testChanRestoreScenario(t *harnessTest, net *lntest.NetworkHarness,
testCase *chanRestoreTestCase, password []byte) {
const (
chanAmt = btcutil.Amount(10000000)
pushAmt = btcutil.Amount(5000000)
)
ctxb := context.Background()
// First, we'll create a brand new node we'll use within the test. If
// we have a custom backup file specified, then we'll also create that
// for use.
dave, mnemonic, err := net.NewNodeWithSeed(
"dave", nil, password,
)
if err != nil {
t.Fatalf("unable to create new node: %v", err)
}
defer shutdownAndAssert(net, t, dave)
carol, err := net.NewNode("carol", nil)
if err != nil {
t.Fatalf("unable to make new node: %v", err)
}
defer shutdownAndAssert(net, t, carol)
// Now that our new node is created, we'll give him some coins it can
// use to open channels with Carol.
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
err = net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, dave)
if err != nil {
t.Fatalf("unable to send coins to dave: %v", err)
}
var from, to *lntest.HarnessNode
if testCase.initiator {
from, to = dave, carol
} else {
from, to = carol, dave
}
// Next, we'll connect Dave to Carol, and open a new channel to her
// with a portion pushed.
if err := net.ConnectNodes(ctxt, dave, carol); err != nil {
t.Fatalf("unable to connect dave to carol: %v", err)
}
ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout)
chanPoint := openChannelAndAssert(
ctxt, t, net, from, to,
lntest.OpenChannelParams{
Amt: chanAmt,
PushAmt: pushAmt,
Private: testCase.private,
},
)
// Wait for both sides to see the opened channel.
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
err = dave.WaitForNetworkChannelOpen(ctxt, chanPoint)
if err != nil {
t.Fatalf("dave didn't report channel: %v", err)
}
err = carol.WaitForNetworkChannelOpen(ctxt, chanPoint)
if err != nil {
t.Fatalf("carol didn't report channel: %v", err)
}
// If both parties should start with existing channel updates, then
// we'll send+settle an HTLC between 'from' and 'to' now.
if testCase.channelsUpdated {
invoice := &lnrpc.Invoice{
Memo: "testing",
Value: 10000,
}
invoiceResp, err := to.AddInvoice(ctxt, invoice)
if err != nil {
t.Fatalf("unable to add invoice: %v", err)
}
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
err = completePaymentRequests(
ctxt, from, []string{invoiceResp.PaymentRequest},
true,
)
if err != nil {
t.Fatalf("unable to complete payments: %v", err)
}
}
// Before we start the recovery, we'll record the balances of both
// Carol and Dave to ensure they both sweep their coins at the end.
balReq := &lnrpc.WalletBalanceRequest{}
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
carolBalResp, err := carol.WalletBalance(ctxt, balReq)
if err != nil {
t.Fatalf("unable to get carol's balance: %v", err)
}
carolStartingBalance := carolBalResp.ConfirmedBalance
daveBalance, err := dave.WalletBalance(ctxt, balReq)
if err != nil {
t.Fatalf("unable to get carol's balance: %v", err)
}
daveStartingBalance := daveBalance.ConfirmedBalance
// At this point, we'll now execute the restore method to give us the
// new node we should attempt our assertions against.
backupFilePath := dave.ChanBackupPath()
restoredNodeFunc, err := testCase.restoreMethod(
dave, backupFilePath, mnemonic,
)
if err != nil {
t.Fatalf("unable to prep node restoration: %v", err)
}
// TODO(roasbeef): assert recovery state in channel
// Now that we're able to make our restored now, we'll shutdown the old
// Dave node as we'll be storing it shortly below.
shutdownAndAssert(net, t, dave)
// Next, we'll make a new Dave and start the bulk of our recovery
// workflow.
dave, err = restoredNodeFunc()
if err != nil {
t.Fatalf("unable to restore node: %v", err)
}
// Now that we have our new node up, we expect that it'll re-connect to
// Carol automatically based on the restored backup.
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
err = net.EnsureConnected(ctxt, dave, carol)
if err != nil {
t.Fatalf("node didn't connect after recovery: %v", err)
}
// TODO(roasbeef): move dave restarts?
// Now we'll assert that both sides properly execute the DLP protocol.
// We grab their balances now to ensure that they're made whole at the
// end of the protocol.
assertDLPExecuted(
net, t, carol, carolStartingBalance, dave, daveStartingBalance,
)
}
// chanRestoreViaRPC is a helper test method that returns a nodeRestorer
// instance which will restore the target node from a password+seed, then
// trigger a SCB restore using the RPC interface.
func chanRestoreViaRPC(net *lntest.NetworkHarness,
password []byte, mnemonic []string,
multi []byte) (nodeRestorer, error) {
backup := &lnrpc.RestoreChanBackupRequest_MultiChanBackup{
MultiChanBackup: multi,
}
ctxb := context.Background()
return func() (*lntest.HarnessNode, error) {
newNode, err := net.RestoreNodeWithSeed(
"dave", nil, password, mnemonic, 1000, nil,
)
if err != nil {
return nil, fmt.Errorf("unable to "+
"restore node: %v", err)
}
_, err = newNode.RestoreChannelBackups(
ctxb, &lnrpc.RestoreChanBackupRequest{
Backup: backup,
},
)
if err != nil {
return nil, fmt.Errorf("unable "+
"to restore backups: %v", err)
}
return newNode, nil
}, nil
}
// testChannelBackupRestore tests that we're able to recover from, and initiate
// the DLP protocol via: the RPC restore command, restoring on unlock, and
// restoring from initial wallet creation. We'll also alternate between
// restoring form the on disk file, and restoring from the exported RPC command
// as well.
func testChannelBackupRestore(net *lntest.NetworkHarness, t *harnessTest) {
password := []byte("El Psy Kongroo")
ctxb := context.Background()
var testCases = []chanRestoreTestCase{
// Restore from backups obtained via the RPC interface. Dave
// was the initiator, of the non-advertised channel.
{
name: "restore from RPC backup",
channelsUpdated: false,
initiator: true,
private: false,
restoreMethod: func(oldNode *lntest.HarnessNode,
backupFilePath string,
mnemonic []string) (nodeRestorer, error) {
// For this restoration method, we'll grab the
// current multi-channel backup from the old
// node, and use it to restore a new node
// within the closure.
req := &lnrpc.ChanBackupExportRequest{}
chanBackup, err := oldNode.ExportAllChannelBackups(
ctxb, req,
)
if err != nil {
return nil, fmt.Errorf("unable to obtain "+
"channel backup: %v", err)
}
multi := chanBackup.MultiChanBackup.MultiChanBackup
// In our nodeRestorer function, we'll restore
// the node from seed, then manually recover
// the channel backup.
return chanRestoreViaRPC(
net, password, mnemonic, multi,
)
},
},
// Restore the backup from the on-disk file, using the RPC
// interface.
{
name: "restore from backup file",
initiator: true,
private: false,
restoreMethod: func(oldNode *lntest.HarnessNode,
backupFilePath string,
mnemonic []string) (nodeRestorer, error) {
// Read the entire Multi backup stored within
// this node's chaannels.backup file.
multi, err := ioutil.ReadFile(backupFilePath)
if err != nil {
return nil, err
}
// Now that we have Dave's backup file, we'll
// create a new nodeRestorer that will restore
// using the on-disk channels.backup.
return chanRestoreViaRPC(
net, password, mnemonic, multi,
)
},
},
// Restore the backup as part of node initialization with the
// prior mnemonic and new backup seed.
{
name: "restore during creation",
initiator: true,
private: false,
restoreMethod: func(oldNode *lntest.HarnessNode,
backupFilePath string,
mnemonic []string) (nodeRestorer, error) {
// First, fetch the current backup state as is,
// to obtain our latest Multi.
chanBackup, err := oldNode.ExportAllChannelBackups(
ctxb, &lnrpc.ChanBackupExportRequest{},
)
if err != nil {
return nil, fmt.Errorf("unable to obtain "+
"channel backup: %v", err)
}
backupSnapshot := &lnrpc.ChanBackupSnapshot{
MultiChanBackup: chanBackup.MultiChanBackup,
}
// Create a new nodeRestorer that will restore
// the node using the Multi backup we just
// obtained above.
return func() (*lntest.HarnessNode, error) {
return net.RestoreNodeWithSeed(
"dave", nil, password,
mnemonic, 1000, backupSnapshot,
)
}, nil
},
},
// Restore the backup once the node has already been
// re-created, using the Unlock call.
{
name: "restore during unlock",
initiator: true,
private: false,
restoreMethod: func(oldNode *lntest.HarnessNode,
backupFilePath string,
mnemonic []string) (nodeRestorer, error) {
// First, fetch the current backup state as is,
// to obtain our latest Multi.
chanBackup, err := oldNode.ExportAllChannelBackups(
ctxb, &lnrpc.ChanBackupExportRequest{},
)
if err != nil {
return nil, fmt.Errorf("unable to obtain "+
"channel backup: %v", err)
}
backupSnapshot := &lnrpc.ChanBackupSnapshot{
MultiChanBackup: chanBackup.MultiChanBackup,
}
// Create a new nodeRestorer that will restore
// the node with its seed, but no channel
// backup, shutdown this initialized node, then
// restart it again using Unlock.
return func() (*lntest.HarnessNode, error) {
newNode, err := net.RestoreNodeWithSeed(
"dave", nil, password,
mnemonic, 1000, nil,
)
if err != nil {
return nil, err
}
err = net.RestartNode(
newNode, nil, backupSnapshot,
)
if err != nil {
return nil, err
}
return newNode, nil
}, nil
},
},
}
// TODO(roasbeef): online vs offline close?
// TODO(roasbeef): need to re-trigger the on-disk file once the node
// ann is updated?
for _, testCase := range testCases {
success := t.t.Run(testCase.name, func(_ *testing.T) {
testChanRestoreScenario(t, net, &testCase, password)
})
if !success {
break
}
}
}
type testCase struct {
name string
test func(net *lntest.NetworkHarness, t *harnessTest)
@ -13680,6 +14078,9 @@ var testsCases = []*testCase{
test: testExportChannelBackup,
},
{
name: "channel backup restore",
test: testChannelBackupRestore,
},
}
// TestLightningNetworkDaemon performs a series of integration tests amongst a