809d238364
To avoid running into the "server is still starting" error when trying to close a channel, we first wait for the error to disappear before we try closing the actual channel.
1106 lines
32 KiB
Go
1106 lines
32 KiB
Go
package itest
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/btcsuite/btcd/wire"
|
|
"github.com/btcsuite/btcutil"
|
|
"github.com/lightningnetwork/lnd/chanbackup"
|
|
"github.com/lightningnetwork/lnd/lnrpc"
|
|
"github.com/lightningnetwork/lnd/lntest"
|
|
"github.com/lightningnetwork/lnd/lntest/wait"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// testChannelBackupRestore tests that we're able to recover from, and initiate
|
|
// the DLP protocol via: the RPC restore command, restoring on unlock, and
|
|
// restoring from initial wallet creation. We'll also alternate between
|
|
// restoring form the on disk file, and restoring from the exported RPC command
|
|
// as well.
|
|
func testChannelBackupRestore(net *lntest.NetworkHarness, t *harnessTest) {
|
|
password := []byte("El Psy Kongroo")
|
|
|
|
ctxb := context.Background()
|
|
|
|
var testCases = []chanRestoreTestCase{
|
|
// Restore from backups obtained via the RPC interface. Dave
|
|
// was the initiator, of the non-advertised channel.
|
|
{
|
|
name: "restore from RPC backup",
|
|
channelsUpdated: false,
|
|
initiator: true,
|
|
private: false,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// For this restoration method, we'll grab the
|
|
// current multi-channel backup from the old
|
|
// node, and use it to restore a new node
|
|
// within the closure.
|
|
req := &lnrpc.ChanBackupExportRequest{}
|
|
chanBackup, err := oldNode.ExportAllChannelBackups(
|
|
ctxb, req,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to obtain "+
|
|
"channel backup: %v", err)
|
|
}
|
|
|
|
multi := chanBackup.MultiChanBackup.MultiChanBackup
|
|
|
|
// In our nodeRestorer function, we'll restore
|
|
// the node from seed, then manually recover
|
|
// the channel backup.
|
|
return chanRestoreViaRPC(
|
|
net, password, mnemonic, multi,
|
|
)
|
|
},
|
|
},
|
|
|
|
// Restore the backup from the on-disk file, using the RPC
|
|
// interface.
|
|
{
|
|
name: "restore from backup file",
|
|
initiator: true,
|
|
private: false,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// Read the entire Multi backup stored within
|
|
// this node's channels.backup file.
|
|
multi, err := ioutil.ReadFile(backupFilePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Now that we have Dave's backup file, we'll
|
|
// create a new nodeRestorer that will restore
|
|
// using the on-disk channels.backup.
|
|
return chanRestoreViaRPC(
|
|
net, password, mnemonic, multi,
|
|
)
|
|
},
|
|
},
|
|
|
|
// Restore the backup as part of node initialization with the
|
|
// prior mnemonic and new backup seed.
|
|
{
|
|
name: "restore during creation",
|
|
initiator: true,
|
|
private: false,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// First, fetch the current backup state as is,
|
|
// to obtain our latest Multi.
|
|
chanBackup, err := oldNode.ExportAllChannelBackups(
|
|
ctxb, &lnrpc.ChanBackupExportRequest{},
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to obtain "+
|
|
"channel backup: %v", err)
|
|
}
|
|
backupSnapshot := &lnrpc.ChanBackupSnapshot{
|
|
MultiChanBackup: chanBackup.MultiChanBackup,
|
|
}
|
|
|
|
// Create a new nodeRestorer that will restore
|
|
// the node using the Multi backup we just
|
|
// obtained above.
|
|
return func() (*lntest.HarnessNode, error) {
|
|
return net.RestoreNodeWithSeed(
|
|
"dave", nil, password,
|
|
mnemonic, 1000, backupSnapshot,
|
|
)
|
|
}, nil
|
|
},
|
|
},
|
|
|
|
// Restore the backup once the node has already been
|
|
// re-created, using the Unlock call.
|
|
{
|
|
name: "restore during unlock",
|
|
initiator: true,
|
|
private: false,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// First, fetch the current backup state as is,
|
|
// to obtain our latest Multi.
|
|
chanBackup, err := oldNode.ExportAllChannelBackups(
|
|
ctxb, &lnrpc.ChanBackupExportRequest{},
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to obtain "+
|
|
"channel backup: %v", err)
|
|
}
|
|
backupSnapshot := &lnrpc.ChanBackupSnapshot{
|
|
MultiChanBackup: chanBackup.MultiChanBackup,
|
|
}
|
|
|
|
// Create a new nodeRestorer that will restore
|
|
// the node with its seed, but no channel
|
|
// backup, shutdown this initialized node, then
|
|
// restart it again using Unlock.
|
|
return func() (*lntest.HarnessNode, error) {
|
|
newNode, err := net.RestoreNodeWithSeed(
|
|
"dave", nil, password,
|
|
mnemonic, 1000, nil,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = net.RestartNode(
|
|
newNode, nil, backupSnapshot,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return newNode, nil
|
|
}, nil
|
|
},
|
|
},
|
|
|
|
// Restore the backup from the on-disk file a second time to
|
|
// make sure imports can be canceled and later resumed.
|
|
{
|
|
name: "restore from backup file twice",
|
|
initiator: true,
|
|
private: false,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// Read the entire Multi backup stored within
|
|
// this node's channels.backup file.
|
|
multi, err := ioutil.ReadFile(backupFilePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Now that we have Dave's backup file, we'll
|
|
// create a new nodeRestorer that will restore
|
|
// using the on-disk channels.backup.
|
|
backup := &lnrpc.RestoreChanBackupRequest_MultiChanBackup{
|
|
MultiChanBackup: multi,
|
|
}
|
|
|
|
ctxb := context.Background()
|
|
|
|
return func() (*lntest.HarnessNode, error) {
|
|
newNode, err := net.RestoreNodeWithSeed(
|
|
"dave", nil, password, mnemonic,
|
|
1000, nil,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to "+
|
|
"restore node: %v", err)
|
|
}
|
|
|
|
_, err = newNode.RestoreChannelBackups(
|
|
ctxb,
|
|
&lnrpc.RestoreChanBackupRequest{
|
|
Backup: backup,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable "+
|
|
"to restore backups: %v",
|
|
err)
|
|
}
|
|
|
|
_, err = newNode.RestoreChannelBackups(
|
|
ctxb,
|
|
&lnrpc.RestoreChanBackupRequest{
|
|
Backup: backup,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable "+
|
|
"to restore backups the"+
|
|
"second time: %v",
|
|
err)
|
|
}
|
|
|
|
return newNode, nil
|
|
}, nil
|
|
},
|
|
},
|
|
|
|
// Use the channel backup file that contains an unconfirmed
|
|
// channel and make sure recovery works as well.
|
|
{
|
|
name: "restore unconfirmed channel file",
|
|
channelsUpdated: false,
|
|
initiator: true,
|
|
private: false,
|
|
unconfirmed: true,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// Read the entire Multi backup stored within
|
|
// this node's channels.backup file.
|
|
multi, err := ioutil.ReadFile(backupFilePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Let's assume time passes, the channel
|
|
// confirms in the meantime but for some reason
|
|
// the backup we made while it was still
|
|
// unconfirmed is the only backup we have. We
|
|
// should still be able to restore it. To
|
|
// simulate time passing, we mine some blocks
|
|
// to get the channel confirmed _after_ we saved
|
|
// the backup.
|
|
mineBlocks(t, net, 6, 1)
|
|
|
|
// In our nodeRestorer function, we'll restore
|
|
// the node from seed, then manually recover
|
|
// the channel backup.
|
|
return chanRestoreViaRPC(
|
|
net, password, mnemonic, multi,
|
|
)
|
|
},
|
|
},
|
|
|
|
// Create a backup using RPC that contains an unconfirmed
|
|
// channel and make sure recovery works as well.
|
|
{
|
|
name: "restore unconfirmed channel RPC",
|
|
channelsUpdated: false,
|
|
initiator: true,
|
|
private: false,
|
|
unconfirmed: true,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// For this restoration method, we'll grab the
|
|
// current multi-channel backup from the old
|
|
// node. The channel should be included, even if
|
|
// it is not confirmed yet.
|
|
req := &lnrpc.ChanBackupExportRequest{}
|
|
chanBackup, err := oldNode.ExportAllChannelBackups(
|
|
ctxb, req,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to obtain "+
|
|
"channel backup: %v", err)
|
|
}
|
|
chanPoints := chanBackup.MultiChanBackup.ChanPoints
|
|
if len(chanPoints) == 0 {
|
|
return nil, fmt.Errorf("unconfirmed " +
|
|
"channel not included in backup")
|
|
}
|
|
|
|
// Let's assume time passes, the channel
|
|
// confirms in the meantime but for some reason
|
|
// the backup we made while it was still
|
|
// unconfirmed is the only backup we have. We
|
|
// should still be able to restore it. To
|
|
// simulate time passing, we mine some blocks
|
|
// to get the channel confirmed _after_ we saved
|
|
// the backup.
|
|
mineBlocks(t, net, 6, 1)
|
|
|
|
// In our nodeRestorer function, we'll restore
|
|
// the node from seed, then manually recover
|
|
// the channel backup.
|
|
multi := chanBackup.MultiChanBackup.MultiChanBackup
|
|
return chanRestoreViaRPC(
|
|
net, password, mnemonic, multi,
|
|
)
|
|
},
|
|
},
|
|
|
|
// Restore the backup from the on-disk file, using the RPC
|
|
// interface, for anchor commitment channels.
|
|
{
|
|
name: "restore from backup file anchors",
|
|
initiator: true,
|
|
private: false,
|
|
anchorCommit: true,
|
|
restoreMethod: func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error) {
|
|
|
|
// Read the entire Multi backup stored within
|
|
// this node's channels.backup file.
|
|
multi, err := ioutil.ReadFile(backupFilePath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Now that we have Dave's backup file, we'll
|
|
// create a new nodeRestorer that will restore
|
|
// using the on-disk channels.backup.
|
|
return chanRestoreViaRPC(
|
|
net, password, mnemonic, multi,
|
|
)
|
|
},
|
|
},
|
|
}
|
|
|
|
// TODO(roasbeef): online vs offline close?
|
|
|
|
// TODO(roasbeef): need to re-trigger the on-disk file once the node
|
|
// ann is updated?
|
|
|
|
for _, testCase := range testCases {
|
|
testCase := testCase
|
|
success := t.t.Run(testCase.name, func(t *testing.T) {
|
|
h := newHarnessTest(t, net)
|
|
|
|
// Start each test with the default static fee estimate.
|
|
net.SetFeeEstimate(12500)
|
|
|
|
testChanRestoreScenario(h, net, &testCase, password)
|
|
})
|
|
if !success {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// testChannelBackupUpdates tests that both the streaming channel update RPC,
|
|
// and the on-disk channels.backup are updated each time a channel is
|
|
// opened/closed.
|
|
func testChannelBackupUpdates(net *lntest.NetworkHarness, t *harnessTest) {
|
|
ctxb := context.Background()
|
|
|
|
// First, we'll make a temp directory that we'll use to store our
|
|
// backup file, so we can check in on it during the test easily.
|
|
backupDir, err := ioutil.TempDir("", "")
|
|
if err != nil {
|
|
t.Fatalf("unable to create backup dir: %v", err)
|
|
}
|
|
defer os.RemoveAll(backupDir)
|
|
|
|
// First, we'll create a new node, Carol. We'll also create a temporary
|
|
// file that Carol will use to store her channel backups.
|
|
backupFilePath := filepath.Join(
|
|
backupDir, chanbackup.DefaultBackupFileName,
|
|
)
|
|
carolArgs := fmt.Sprintf("--backupfilepath=%v", backupFilePath)
|
|
carol, err := net.NewNode("carol", []string{carolArgs})
|
|
if err != nil {
|
|
t.Fatalf("unable to create new node: %v", err)
|
|
}
|
|
defer shutdownAndAssert(net, t, carol)
|
|
|
|
// Next, we'll register for streaming notifications for changes to the
|
|
// backup file.
|
|
backupStream, err := carol.SubscribeChannelBackups(
|
|
ctxb, &lnrpc.ChannelBackupSubscription{},
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("unable to create backup stream: %v", err)
|
|
}
|
|
|
|
// We'll use this goroutine to proxy any updates to a channel we can
|
|
// easily use below.
|
|
var wg sync.WaitGroup
|
|
backupUpdates := make(chan *lnrpc.ChanBackupSnapshot)
|
|
streamErr := make(chan error)
|
|
streamQuit := make(chan struct{})
|
|
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for {
|
|
snapshot, err := backupStream.Recv()
|
|
if err != nil {
|
|
select {
|
|
case streamErr <- err:
|
|
case <-streamQuit:
|
|
return
|
|
}
|
|
}
|
|
|
|
select {
|
|
case backupUpdates <- snapshot:
|
|
case <-streamQuit:
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
defer close(streamQuit)
|
|
|
|
// With Carol up, we'll now connect her to Alice, and open a channel
|
|
// between them.
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
if err := net.ConnectNodes(ctxt, carol, net.Alice); err != nil {
|
|
t.Fatalf("unable to connect carol to alice: %v", err)
|
|
}
|
|
|
|
// Next, we'll open two channels between Alice and Carol back to back.
|
|
var chanPoints []*lnrpc.ChannelPoint
|
|
numChans := 2
|
|
chanAmt := btcutil.Amount(1000000)
|
|
for i := 0; i < numChans; i++ {
|
|
ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout)
|
|
chanPoint := openChannelAndAssert(
|
|
ctxt, t, net, net.Alice, carol,
|
|
lntest.OpenChannelParams{
|
|
Amt: chanAmt,
|
|
},
|
|
)
|
|
|
|
chanPoints = append(chanPoints, chanPoint)
|
|
}
|
|
|
|
// Using this helper function, we'll maintain a pointer to the latest
|
|
// channel backup so we can compare it to the on disk state.
|
|
var currentBackup *lnrpc.ChanBackupSnapshot
|
|
assertBackupNtfns := func(numNtfns int) {
|
|
for i := 0; i < numNtfns; i++ {
|
|
select {
|
|
case err := <-streamErr:
|
|
t.Fatalf("error with backup stream: %v", err)
|
|
|
|
case currentBackup = <-backupUpdates:
|
|
|
|
case <-time.After(time.Second * 5):
|
|
t.Fatalf("didn't receive channel backup "+
|
|
"notification %v", i+1)
|
|
}
|
|
}
|
|
}
|
|
|
|
// assertBackupFileState is a helper function that we'll use to compare
|
|
// the on disk back up file to our currentBackup pointer above.
|
|
assertBackupFileState := func() {
|
|
err := wait.NoError(func() error {
|
|
packedBackup, err := ioutil.ReadFile(backupFilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to read backup "+
|
|
"file: %v", err)
|
|
}
|
|
|
|
// As each back up file will be encrypted with a fresh
|
|
// nonce, we can't compare them directly, so instead
|
|
// we'll compare the length which is a proxy for the
|
|
// number of channels that the multi-backup contains.
|
|
rawBackup := currentBackup.MultiChanBackup.MultiChanBackup
|
|
if len(rawBackup) != len(packedBackup) {
|
|
return fmt.Errorf("backup files don't match: "+
|
|
"expected %x got %x", rawBackup, packedBackup)
|
|
}
|
|
|
|
// Additionally, we'll assert that both backups up
|
|
// returned are valid.
|
|
for i, backup := range [][]byte{rawBackup, packedBackup} {
|
|
snapshot := &lnrpc.ChanBackupSnapshot{
|
|
MultiChanBackup: &lnrpc.MultiChanBackup{
|
|
MultiChanBackup: backup,
|
|
},
|
|
}
|
|
_, err := carol.VerifyChanBackup(ctxb, snapshot)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to verify "+
|
|
"backup #%d: %v", i, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}, defaultTimeout)
|
|
if err != nil {
|
|
t.Fatalf("backup state invalid: %v", err)
|
|
}
|
|
}
|
|
|
|
// As these two channels were just opened, we should've got two times
|
|
// the pending and open notifications for channel backups.
|
|
assertBackupNtfns(2 * 2)
|
|
|
|
// The on disk file should also exactly match the latest backup that we
|
|
// have.
|
|
assertBackupFileState()
|
|
|
|
// Next, we'll close the channels one by one. After each channel
|
|
// closure, we should get a notification, and the on-disk state should
|
|
// match this state as well.
|
|
for i := 0; i < numChans; i++ {
|
|
// To ensure force closes also trigger an update, we'll force
|
|
// close half of the channels.
|
|
forceClose := i%2 == 0
|
|
|
|
chanPoint := chanPoints[i]
|
|
|
|
ctxt, _ := context.WithTimeout(ctxb, channelCloseTimeout)
|
|
closeChannelAndAssert(
|
|
ctxt, t, net, net.Alice, chanPoint, forceClose,
|
|
)
|
|
|
|
// We should get a single notification after closing, and the
|
|
// on-disk state should match this latest notifications.
|
|
assertBackupNtfns(1)
|
|
assertBackupFileState()
|
|
|
|
// If we force closed the channel, then we'll mine enough
|
|
// blocks to ensure all outputs have been swept.
|
|
if forceClose {
|
|
cleanupForceClose(t, net, net.Alice, chanPoint)
|
|
}
|
|
}
|
|
}
|
|
|
|
// testExportChannelBackup tests that we're able to properly export either a
|
|
// targeted channel's backup, or export backups of all the currents open
|
|
// channels.
|
|
func testExportChannelBackup(net *lntest.NetworkHarness, t *harnessTest) {
|
|
ctxb := context.Background()
|
|
|
|
// First, we'll create our primary test node: Carol. We'll use Carol to
|
|
// open channels and also export backups that we'll examine throughout
|
|
// the test.
|
|
carol, err := net.NewNode("carol", nil)
|
|
if err != nil {
|
|
t.Fatalf("unable to create new node: %v", err)
|
|
}
|
|
defer shutdownAndAssert(net, t, carol)
|
|
|
|
// With Carol up, we'll now connect her to Alice, and open a channel
|
|
// between them.
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
if err := net.ConnectNodes(ctxt, carol, net.Alice); err != nil {
|
|
t.Fatalf("unable to connect carol to alice: %v", err)
|
|
}
|
|
|
|
// Next, we'll open two channels between Alice and Carol back to back.
|
|
var chanPoints []*lnrpc.ChannelPoint
|
|
numChans := 2
|
|
chanAmt := btcutil.Amount(1000000)
|
|
for i := 0; i < numChans; i++ {
|
|
ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout)
|
|
chanPoint := openChannelAndAssert(
|
|
ctxt, t, net, net.Alice, carol,
|
|
lntest.OpenChannelParams{
|
|
Amt: chanAmt,
|
|
},
|
|
)
|
|
|
|
chanPoints = append(chanPoints, chanPoint)
|
|
}
|
|
|
|
// Now that the channels are open, we should be able to fetch the
|
|
// backups of each of the channels.
|
|
for _, chanPoint := range chanPoints {
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
req := &lnrpc.ExportChannelBackupRequest{
|
|
ChanPoint: chanPoint,
|
|
}
|
|
chanBackup, err := carol.ExportChannelBackup(ctxt, req)
|
|
if err != nil {
|
|
t.Fatalf("unable to fetch backup for channel %v: %v",
|
|
chanPoint, err)
|
|
}
|
|
|
|
// The returned backup should be full populated. Since it's
|
|
// encrypted, we can't assert any more than that atm.
|
|
if len(chanBackup.ChanBackup) == 0 {
|
|
t.Fatalf("obtained empty backup for channel: %v", chanPoint)
|
|
}
|
|
|
|
// The specified chanPoint in the response should match our
|
|
// requested chanPoint.
|
|
if chanBackup.ChanPoint.String() != chanPoint.String() {
|
|
t.Fatalf("chanPoint mismatched: expected %v, got %v",
|
|
chanPoint.String(),
|
|
chanBackup.ChanPoint.String())
|
|
}
|
|
}
|
|
|
|
// Before we proceed, we'll make two utility methods we'll use below
|
|
// for our primary assertions.
|
|
assertNumSingleBackups := func(numSingles int) {
|
|
err := wait.NoError(func() error {
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
req := &lnrpc.ChanBackupExportRequest{}
|
|
chanSnapshot, err := carol.ExportAllChannelBackups(
|
|
ctxt, req,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to export channel "+
|
|
"backup: %v", err)
|
|
}
|
|
|
|
if chanSnapshot.SingleChanBackups == nil {
|
|
return fmt.Errorf("single chan backups not " +
|
|
"populated")
|
|
}
|
|
|
|
backups := chanSnapshot.SingleChanBackups.ChanBackups
|
|
if len(backups) != numSingles {
|
|
return fmt.Errorf("expected %v singles, "+
|
|
"got %v", len(backups), numSingles)
|
|
}
|
|
|
|
return nil
|
|
}, defaultTimeout)
|
|
if err != nil {
|
|
t.Fatalf(err.Error())
|
|
}
|
|
}
|
|
assertMultiBackupFound := func() func(bool, map[wire.OutPoint]struct{}) {
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
req := &lnrpc.ChanBackupExportRequest{}
|
|
chanSnapshot, err := carol.ExportAllChannelBackups(ctxt, req)
|
|
if err != nil {
|
|
t.Fatalf("unable to export channel backup: %v", err)
|
|
}
|
|
|
|
return func(found bool, chanPoints map[wire.OutPoint]struct{}) {
|
|
switch {
|
|
case found && chanSnapshot.MultiChanBackup == nil:
|
|
t.Fatalf("multi-backup not present")
|
|
|
|
case !found && chanSnapshot.MultiChanBackup != nil &&
|
|
(len(chanSnapshot.MultiChanBackup.MultiChanBackup) !=
|
|
chanbackup.NilMultiSizePacked):
|
|
|
|
t.Fatalf("found multi-backup when non should " +
|
|
"be found")
|
|
}
|
|
|
|
if !found {
|
|
return
|
|
}
|
|
|
|
backedUpChans := chanSnapshot.MultiChanBackup.ChanPoints
|
|
if len(chanPoints) != len(backedUpChans) {
|
|
t.Fatalf("expected %v chans got %v", len(chanPoints),
|
|
len(backedUpChans))
|
|
}
|
|
|
|
for _, chanPoint := range backedUpChans {
|
|
wirePoint := rpcPointToWirePoint(t, chanPoint)
|
|
if _, ok := chanPoints[wirePoint]; !ok {
|
|
t.Fatalf("unexpected backup: %v", wirePoint)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
chans := make(map[wire.OutPoint]struct{})
|
|
for _, chanPoint := range chanPoints {
|
|
chans[rpcPointToWirePoint(t, chanPoint)] = struct{}{}
|
|
}
|
|
|
|
// We should have exactly two single channel backups contained, and we
|
|
// should also have a multi-channel backup.
|
|
assertNumSingleBackups(2)
|
|
assertMultiBackupFound()(true, chans)
|
|
|
|
// We'll now close each channel on by one. After we close a channel, we
|
|
// shouldn't be able to find that channel as a backup still. We should
|
|
// also have one less single written to disk.
|
|
for i, chanPoint := range chanPoints {
|
|
ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout)
|
|
closeChannelAndAssert(
|
|
ctxt, t, net, net.Alice, chanPoint, false,
|
|
)
|
|
|
|
assertNumSingleBackups(len(chanPoints) - i - 1)
|
|
|
|
delete(chans, rpcPointToWirePoint(t, chanPoint))
|
|
assertMultiBackupFound()(true, chans)
|
|
}
|
|
|
|
// At this point we shouldn't have any single or multi-chan backups at
|
|
// all.
|
|
assertNumSingleBackups(0)
|
|
assertMultiBackupFound()(false, nil)
|
|
}
|
|
|
|
// nodeRestorer is a function closure that allows each chanRestoreTestCase to
|
|
// control exactly *how* the prior node is restored. This might be using an
|
|
// backup obtained over RPC, or the file system, etc.
|
|
type nodeRestorer func() (*lntest.HarnessNode, error)
|
|
|
|
// chanRestoreTestCase describes a test case for an end to end SCB restoration
|
|
// work flow. One node will start from scratch using an existing SCB. At the
|
|
// end of the est, both nodes should be made whole via the DLP protocol.
|
|
type chanRestoreTestCase struct {
|
|
// name is the name of the target test case.
|
|
name string
|
|
|
|
// channelsUpdated is false then this means that no updates
|
|
// have taken place within the channel before restore.
|
|
// Otherwise, HTLCs will be settled between the two parties
|
|
// before restoration modifying the balance beyond the initial
|
|
// allocation.
|
|
channelsUpdated bool
|
|
|
|
// initiator signals if Dave should be the one that opens the
|
|
// channel to Alice, or if it should be the other way around.
|
|
initiator bool
|
|
|
|
// private signals if the channel from Dave to Carol should be
|
|
// private or not.
|
|
private bool
|
|
|
|
// unconfirmed signals if the channel from Dave to Carol should be
|
|
// confirmed or not.
|
|
unconfirmed bool
|
|
|
|
// anchorCommit is true, then the new anchor commitment type will be
|
|
// used for the channels created in the test.
|
|
anchorCommit bool
|
|
|
|
// restoreMethod takes an old node, then returns a function
|
|
// closure that'll return the same node, but with its state
|
|
// restored via a custom method. We use this to abstract away
|
|
// _how_ a node is restored from our assertions once the node
|
|
// has been fully restored itself.
|
|
restoreMethod func(oldNode *lntest.HarnessNode,
|
|
backupFilePath string,
|
|
mnemonic []string) (nodeRestorer, error)
|
|
}
|
|
|
|
// testChanRestoreScenario executes a chanRestoreTestCase from end to end,
|
|
// ensuring that after Dave restores his channel state according to the
|
|
// testCase, the DLP protocol is executed properly and both nodes are made
|
|
// whole.
|
|
func testChanRestoreScenario(t *harnessTest, net *lntest.NetworkHarness,
|
|
testCase *chanRestoreTestCase, password []byte) {
|
|
|
|
const (
|
|
chanAmt = btcutil.Amount(10000000)
|
|
pushAmt = btcutil.Amount(5000000)
|
|
)
|
|
|
|
ctxb := context.Background()
|
|
|
|
var nodeArgs []string
|
|
if testCase.anchorCommit {
|
|
nodeArgs = commitTypeAnchors.Args()
|
|
}
|
|
|
|
// First, we'll create a brand new node we'll use within the test. If
|
|
// we have a custom backup file specified, then we'll also create that
|
|
// for use.
|
|
dave, mnemonic, _, err := net.NewNodeWithSeed(
|
|
"dave", nodeArgs, password, false,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("unable to create new node: %v", err)
|
|
}
|
|
// Defer to a closure instead of to shutdownAndAssert due to the value
|
|
// of 'dave' changing throughout the test.
|
|
defer func() {
|
|
shutdownAndAssert(net, t, dave)
|
|
}()
|
|
carol, err := net.NewNode("carol", nodeArgs)
|
|
if err != nil {
|
|
t.Fatalf("unable to make new node: %v", err)
|
|
}
|
|
defer shutdownAndAssert(net, t, carol)
|
|
|
|
// Now that our new nodes are created, we'll give them some coins for
|
|
// channel opening and anchor sweeping.
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
err = net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, carol)
|
|
if err != nil {
|
|
t.Fatalf("unable to send coins to dave: %v", err)
|
|
}
|
|
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
|
|
err = net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, dave)
|
|
if err != nil {
|
|
t.Fatalf("unable to send coins to dave: %v", err)
|
|
}
|
|
|
|
var from, to *lntest.HarnessNode
|
|
if testCase.initiator {
|
|
from, to = dave, carol
|
|
} else {
|
|
from, to = carol, dave
|
|
}
|
|
|
|
// Next, we'll connect Dave to Carol, and open a new channel to her
|
|
// with a portion pushed.
|
|
if err := net.ConnectNodes(ctxt, dave, carol); err != nil {
|
|
t.Fatalf("unable to connect dave to carol: %v", err)
|
|
}
|
|
|
|
// We will either open a confirmed or unconfirmed channel, depending on
|
|
// the requirements of the test case.
|
|
switch {
|
|
case testCase.unconfirmed:
|
|
ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout)
|
|
_, err := net.OpenPendingChannel(
|
|
ctxt, from, to, chanAmt, pushAmt,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("couldn't open pending channel: %v", err)
|
|
}
|
|
|
|
// Give the pubsub some time to update the channel backup.
|
|
err = wait.NoError(func() error {
|
|
fi, err := os.Stat(dave.ChanBackupPath())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if fi.Size() <= chanbackup.NilMultiSizePacked {
|
|
return fmt.Errorf("backup file empty")
|
|
}
|
|
return nil
|
|
}, defaultTimeout)
|
|
if err != nil {
|
|
t.Fatalf("channel backup not updated in time: %v", err)
|
|
}
|
|
|
|
default:
|
|
ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout)
|
|
chanPoint := openChannelAndAssert(
|
|
ctxt, t, net, from, to,
|
|
lntest.OpenChannelParams{
|
|
Amt: chanAmt,
|
|
PushAmt: pushAmt,
|
|
Private: testCase.private,
|
|
},
|
|
)
|
|
|
|
// Wait for both sides to see the opened channel.
|
|
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
|
|
err = dave.WaitForNetworkChannelOpen(ctxt, chanPoint)
|
|
if err != nil {
|
|
t.Fatalf("dave didn't report channel: %v", err)
|
|
}
|
|
err = carol.WaitForNetworkChannelOpen(ctxt, chanPoint)
|
|
if err != nil {
|
|
t.Fatalf("carol didn't report channel: %v", err)
|
|
}
|
|
}
|
|
|
|
// If both parties should start with existing channel updates, then
|
|
// we'll send+settle an HTLC between 'from' and 'to' now.
|
|
if testCase.channelsUpdated {
|
|
invoice := &lnrpc.Invoice{
|
|
Memo: "testing",
|
|
Value: 10000,
|
|
}
|
|
invoiceResp, err := to.AddInvoice(ctxt, invoice)
|
|
if err != nil {
|
|
t.Fatalf("unable to add invoice: %v", err)
|
|
}
|
|
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
err = completePaymentRequests(
|
|
ctxt, from, from.RouterClient,
|
|
[]string{invoiceResp.PaymentRequest}, true,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("unable to complete payments: %v", err)
|
|
}
|
|
}
|
|
|
|
// Before we start the recovery, we'll record the balances of both
|
|
// Carol and Dave to ensure they both sweep their coins at the end.
|
|
balReq := &lnrpc.WalletBalanceRequest{}
|
|
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
|
|
carolBalResp, err := carol.WalletBalance(ctxt, balReq)
|
|
if err != nil {
|
|
t.Fatalf("unable to get carol's balance: %v", err)
|
|
}
|
|
carolStartingBalance := carolBalResp.ConfirmedBalance
|
|
|
|
daveBalance, err := dave.WalletBalance(ctxt, balReq)
|
|
if err != nil {
|
|
t.Fatalf("unable to get carol's balance: %v", err)
|
|
}
|
|
daveStartingBalance := daveBalance.ConfirmedBalance
|
|
|
|
// At this point, we'll now execute the restore method to give us the
|
|
// new node we should attempt our assertions against.
|
|
backupFilePath := dave.ChanBackupPath()
|
|
restoredNodeFunc, err := testCase.restoreMethod(
|
|
dave, backupFilePath, mnemonic,
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("unable to prep node restoration: %v", err)
|
|
}
|
|
|
|
// Now that we're able to make our restored now, we'll shutdown the old
|
|
// Dave node as we'll be storing it shortly below.
|
|
shutdownAndAssert(net, t, dave)
|
|
|
|
// To make sure the channel state is advanced correctly if the channel
|
|
// peer is not online at first, we also shutdown Carol.
|
|
restartCarol, err := net.SuspendNode(carol)
|
|
require.NoError(t.t, err)
|
|
|
|
// Next, we'll make a new Dave and start the bulk of our recovery
|
|
// workflow.
|
|
dave, err = restoredNodeFunc()
|
|
if err != nil {
|
|
t.Fatalf("unable to restore node: %v", err)
|
|
}
|
|
|
|
// First ensure that the on-chain balance is restored.
|
|
err = wait.NoError(func() error {
|
|
ctxt, _ := context.WithTimeout(ctxb, defaultTimeout)
|
|
balReq := &lnrpc.WalletBalanceRequest{}
|
|
daveBalResp, err := dave.WalletBalance(ctxt, balReq)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
daveBal := daveBalResp.ConfirmedBalance
|
|
if daveBal <= 0 {
|
|
return fmt.Errorf("expected positive balance, had %v",
|
|
daveBal)
|
|
}
|
|
|
|
return nil
|
|
}, defaultTimeout)
|
|
if err != nil {
|
|
t.Fatalf("On-chain balance not restored: %v", err)
|
|
}
|
|
|
|
// We now check that the restored channel is in the proper state. It
|
|
// should not yet be force closing as no connection with the remote
|
|
// peer was established yet. We should also not be able to close the
|
|
// channel.
|
|
assertNumPendingChannels(t, dave, 1, 0)
|
|
ctxt, cancel := context.WithTimeout(ctxb, defaultTimeout)
|
|
defer cancel()
|
|
pendingChanResp, err := dave.PendingChannels(
|
|
ctxt, &lnrpc.PendingChannelsRequest{},
|
|
)
|
|
require.NoError(t.t, err)
|
|
|
|
// We now need to make sure the server is fully started before we can
|
|
// actually close the channel. This is the first check in CloseChannel
|
|
// so we can try with a nil channel point until we get the correct error
|
|
// to find out if Dave is fully started.
|
|
err = wait.Predicate(func() bool {
|
|
const expectedErr = "must specify channel point"
|
|
ctxc, cancel := context.WithCancel(ctxt)
|
|
defer cancel()
|
|
|
|
resp, err := dave.CloseChannel(
|
|
ctxc, &lnrpc.CloseChannelRequest{},
|
|
)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
defer func() { _ = resp.CloseSend() }()
|
|
|
|
_, err = resp.Recv()
|
|
if err != nil && strings.Contains(err.Error(), expectedErr) {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}, defaultTimeout)
|
|
require.NoError(t.t, err)
|
|
|
|
// We also want to make sure we cannot force close in this state. That
|
|
// would get the state machine in a weird state.
|
|
chanPointParts := strings.Split(
|
|
pendingChanResp.WaitingCloseChannels[0].Channel.ChannelPoint,
|
|
":",
|
|
)
|
|
chanPointIndex, _ := strconv.ParseUint(chanPointParts[1], 10, 32)
|
|
resp, err := dave.CloseChannel(ctxt, &lnrpc.CloseChannelRequest{
|
|
ChannelPoint: &lnrpc.ChannelPoint{
|
|
FundingTxid: &lnrpc.ChannelPoint_FundingTxidStr{
|
|
FundingTxidStr: chanPointParts[0],
|
|
},
|
|
OutputIndex: uint32(chanPointIndex),
|
|
},
|
|
Force: true,
|
|
})
|
|
|
|
// We don't get an error directly but only when reading the first
|
|
// message of the stream.
|
|
require.NoError(t.t, err)
|
|
_, err = resp.Recv()
|
|
require.Error(t.t, err)
|
|
require.Contains(t.t, err.Error(), "cannot close channel with state: ")
|
|
require.Contains(t.t, err.Error(), "ChanStatusRestored")
|
|
|
|
// Increase the fee estimate so that the following force close tx will
|
|
// be cpfp'ed in case of anchor commitments.
|
|
net.SetFeeEstimate(30000)
|
|
|
|
// Now that we have ensured that the channels restored by the backup are
|
|
// in the correct state even without the remote peer telling us so,
|
|
// let's start up Carol again.
|
|
err = restartCarol()
|
|
require.NoError(t.t, err)
|
|
|
|
// Now that we have our new node up, we expect that it'll re-connect to
|
|
// Carol automatically based on the restored backup.
|
|
ctxt, _ = context.WithTimeout(ctxb, defaultTimeout)
|
|
err = net.EnsureConnected(ctxt, dave, carol)
|
|
if err != nil {
|
|
t.Fatalf("node didn't connect after recovery: %v", err)
|
|
}
|
|
|
|
// TODO(roasbeef): move dave restarts?
|
|
|
|
// Now we'll assert that both sides properly execute the DLP protocol.
|
|
// We grab their balances now to ensure that they're made whole at the
|
|
// end of the protocol.
|
|
assertDLPExecuted(
|
|
net, t, carol, carolStartingBalance, dave, daveStartingBalance,
|
|
testCase.anchorCommit,
|
|
)
|
|
}
|
|
|
|
// chanRestoreViaRPC is a helper test method that returns a nodeRestorer
|
|
// instance which will restore the target node from a password+seed, then
|
|
// trigger a SCB restore using the RPC interface.
|
|
func chanRestoreViaRPC(net *lntest.NetworkHarness,
|
|
password []byte, mnemonic []string,
|
|
multi []byte) (nodeRestorer, error) {
|
|
|
|
backup := &lnrpc.RestoreChanBackupRequest_MultiChanBackup{
|
|
MultiChanBackup: multi,
|
|
}
|
|
|
|
ctxb := context.Background()
|
|
|
|
return func() (*lntest.HarnessNode, error) {
|
|
newNode, err := net.RestoreNodeWithSeed(
|
|
"dave", nil, password, mnemonic, 1000, nil,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to "+
|
|
"restore node: %v", err)
|
|
}
|
|
|
|
_, err = newNode.RestoreChannelBackups(
|
|
ctxb, &lnrpc.RestoreChanBackupRequest{
|
|
Backup: backup,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable "+
|
|
"to restore backups: %v", err)
|
|
}
|
|
|
|
return newNode, nil
|
|
}, nil
|
|
}
|