From 4d2a12e5523d5409fd5e5004d4c1af9e27c4da7a Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 16:26:57 +0100 Subject: [PATCH 1/7] itest: fix incorrect error message --- lntest/itest/lnd_multi-hop_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lntest/itest/lnd_multi-hop_test.go b/lntest/itest/lnd_multi-hop_test.go index 3707ae11..a7086310 100644 --- a/lntest/itest/lnd_multi-hop_test.go +++ b/lntest/itest/lnd_multi-hop_test.go @@ -272,7 +272,7 @@ func createThreeHopNetwork(t *harnessTest, net *lntest.NetworkHarness, ctxt, _ = context.WithTimeout(context.Background(), defaultTimeout) err = net.SendCoins(ctxt, btcutil.SatoshiPerBitcoin, carol) if err != nil { - t.Fatalf("unable to send coins to Alice: %v", err) + t.Fatalf("unable to send coins to Carol: %v", err) } } From 38233158206de92bb04df77da9878a7251119871 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 16:26:59 +0100 Subject: [PATCH 2/7] lntest: go easy on goroutines when polling In high CPU usage scenarios such as our parallel itests, it seems that some goroutines just don't get any CPU time before our test timeouts expire. By polling 10 times less frequently, we hope to reduce the overall number of goroutines that are spawned because of the RPC requests within the polling code. --- lntest/harness.go | 2 +- lntest/wait/wait.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lntest/harness.go b/lntest/harness.go index 2b876116..cd77ffd2 100644 --- a/lntest/harness.go +++ b/lntest/harness.go @@ -223,7 +223,7 @@ func (n *NetworkHarness) SetUp(testCase string, lndArgs []string) error { // Now block until both wallets have fully synced up. expectedBalance := int64(btcutil.SatoshiPerBitcoin * 10) balReq := &lnrpc.WalletBalanceRequest{} - balanceTicker := time.NewTicker(time.Millisecond * 50) + balanceTicker := time.NewTicker(time.Millisecond * 200) defer balanceTicker.Stop() balanceTimeout := time.After(time.Second * 30) out: diff --git a/lntest/wait/wait.go b/lntest/wait/wait.go index 88cdb27d..a2981816 100644 --- a/lntest/wait/wait.go +++ b/lntest/wait/wait.go @@ -11,7 +11,7 @@ import ( // several running lnd nodes. This function gives callers a way to assert that // some property is upheld within a particular time frame. func Predicate(pred func() bool, timeout time.Duration) error { - const pollInterval = 20 * time.Millisecond + const pollInterval = 200 * time.Millisecond exitTimer := time.After(timeout) for { From dfb131f82fcace3aebf4156f093fa9ecbd6f5056 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 16:27:00 +0100 Subject: [PATCH 3/7] make: reduce itest parallelism from 6 to 4 It seems that our itests don't perform correctly in a high CPU usage scenario such as running 6 tests in parallel. Some goroutines don't get enough execution time and causes check to time out. By reducing the total number of parallel tests, we hope to give all goroutines some more breathing room. --- make/testing_flags.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make/testing_flags.mk b/make/testing_flags.mk index f8293c1c..da5cbeb1 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -5,7 +5,7 @@ TEST_FLAGS = ITEST_FLAGS = EXEC_SUFFIX = COVER_PKG = $$(go list -deps ./... | grep '$(PKG)' | grep -v lnrpc) -NUM_ITEST_TRANCHES = 6 +NUM_ITEST_TRANCHES = 4 ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES) # If rpc option is set also add all extra RPC tags to DEV_TAGS From cdcbc0376d27f62d67210d0fa55451d3f0e9ff31 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 16:27:01 +0100 Subject: [PATCH 4/7] lntest: replace hard coded timeouts This commit replaces most of the hard coded 10, 15, 20 and 30 second timeouts with the default timeout. This should allow darwin users to successfully run the parallel itests locally as well. --- lntest/harness.go | 22 ++--- lntest/itest/lnd_channel_backup_test.go | 2 +- lntest/itest/lnd_test.go | 118 ++++++++++++------------ lntest/node.go | 6 +- 4 files changed, 74 insertions(+), 74 deletions(-) diff --git a/lntest/harness.go b/lntest/harness.go index cd77ffd2..35391bad 100644 --- a/lntest/harness.go +++ b/lntest/harness.go @@ -225,7 +225,7 @@ func (n *NetworkHarness) SetUp(testCase string, lndArgs []string) error { balReq := &lnrpc.WalletBalanceRequest{} balanceTicker := time.NewTicker(time.Millisecond * 200) defer balanceTicker.Stop() - balanceTimeout := time.After(time.Second * 30) + balanceTimeout := time.After(DefaultTimeout) out: for { select { @@ -290,7 +290,7 @@ func (n *NetworkHarness) NewNodeWithSeed(name string, extraArgs []string, return nil, nil, nil, err } - timeout := time.Duration(time.Second * 15) + timeout := time.Duration(DefaultTimeout) ctxb := context.Background() // Create a request to generate a new aezeed. The new seed will have the @@ -424,7 +424,7 @@ func (n *NetworkHarness) RegisterNode(node *HarnessNode) { func (n *NetworkHarness) connect(ctx context.Context, req *lnrpc.ConnectPeerRequest, a *HarnessNode) error { - syncTimeout := time.After(15 * time.Second) + syncTimeout := time.After(DefaultTimeout) tryconnect: if _, err := a.ConnectPeer(ctx, req); err != nil { // If the chain backend is still syncing, retry. @@ -457,7 +457,7 @@ func (n *NetworkHarness) EnsureConnected(ctx context.Context, a, b *HarnessNode) errConnectionRequested := errors.New("connection request in progress") tryConnect := func(a, b *HarnessNode) error { - ctxt, _ := context.WithTimeout(ctx, 15*time.Second) + ctxt, _ := context.WithTimeout(ctx, DefaultTimeout) bInfo, err := b.GetInfo(ctxt, &lnrpc.GetInfoRequest{}) if err != nil { return err @@ -472,7 +472,7 @@ func (n *NetworkHarness) EnsureConnected(ctx context.Context, a, b *HarnessNode) var predErr error err = wait.Predicate(func() bool { - ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + ctx, cancel := context.WithTimeout(ctx, DefaultTimeout) defer cancel() err := n.connect(ctx, req, a) @@ -530,7 +530,7 @@ func (n *NetworkHarness) EnsureConnected(ctx context.Context, a, b *HarnessNode) // If node B is seen in the ListPeers response from node A, // then we can exit early as the connection has been fully // established. - ctxt, _ := context.WithTimeout(ctx, 15*time.Second) + ctxt, _ := context.WithTimeout(ctx, DefaultTimeout) resp, err := b.ListPeers(ctxt, &lnrpc.ListPeersRequest{}) if err != nil { return false @@ -547,7 +547,7 @@ func (n *NetworkHarness) EnsureConnected(ctx context.Context, a, b *HarnessNode) err := wait.Predicate(func() bool { return findSelfInPeerList(a, b) && findSelfInPeerList(b, a) - }, time.Second*15) + }, DefaultTimeout) if err != nil { return fmt.Errorf("peers not connected within 15 seconds") } @@ -594,7 +594,7 @@ func (n *NetworkHarness) ConnectNodes(ctx context.Context, a, b *HarnessNode) er } return false - }, time.Second*15) + }, DefaultTimeout) if err != nil { return fmt.Errorf("peers not connected within 15 seconds") } @@ -1038,7 +1038,7 @@ func (n *NetworkHarness) CloseChannel(ctx context.Context, // We'll wait for *both* nodes to read the channel as active if we're // performing a cooperative channel closure. if !force { - timeout := time.Second * 15 + timeout := DefaultTimeout listReq := &lnrpc.ListChannelsRequest{} // We define two helper functions, one two locate a particular @@ -1223,7 +1223,7 @@ func (n *NetworkHarness) AssertChannelExists(ctx context.Context, } return fmt.Errorf("channel %s not found", chanPoint) - }, 15*time.Second) + }, DefaultTimeout) } // DumpLogs reads the current logs generated by the passed node, and returns @@ -1355,7 +1355,7 @@ func (n *NetworkHarness) sendCoins(ctx context.Context, amt btcutil.Amount, } return nil - }, 15*time.Second) + }, DefaultTimeout) if err != nil { return fmt.Errorf("unconfirmed utxo was not found in "+ "ListUnspent: %v", err) diff --git a/lntest/itest/lnd_channel_backup_test.go b/lntest/itest/lnd_channel_backup_test.go index e6b57a73..5c650ff4 100644 --- a/lntest/itest/lnd_channel_backup_test.go +++ b/lntest/itest/lnd_channel_backup_test.go @@ -520,7 +520,7 @@ func testChannelBackupUpdates(net *lntest.NetworkHarness, t *harnessTest) { } return nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("backup state invalid: %v", err) } diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 04c5a10e..7cd69448 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -373,7 +373,7 @@ func assertChannelClosed(ctx context.Context, t *harnessTest, } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("closing transaction not marked as fully closed") } @@ -532,7 +532,7 @@ func assertNumOpenChannelsPending(ctxt context.Context, t *harnessTest, } return nil - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(err.Error()) } @@ -703,7 +703,7 @@ func completePaymentRequests(ctx context.Context, client lnrpc.LightningClient, } return false - }, time.Second*15) + }, defaultTimeout) if err != nil { return err } @@ -848,7 +848,7 @@ func testGetRecoveryInfo(net *lntest.NetworkHarness, t *harnessTest) { } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("expected recovery mode to be %v, got %v, "+ "expected recovery finished to be %v, got %v, "+ @@ -952,7 +952,7 @@ func testOnchainFundRecovery(net *lntest.NetworkHarness, t *harnessTest) { } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("expected restored node to have %d satoshis, "+ "instead has %d satoshis, expected %d utxos "+ @@ -1805,7 +1805,7 @@ out: } case err := <-subscription.errChan: t.Fatalf("unable to recv graph update: %v", err) - case <-time.After(20 * time.Second): + case <-time.After(defaultTimeout): t.Fatalf("did not receive channel update") } } @@ -2396,7 +2396,7 @@ func waitForNodeBlockHeight(ctx context.Context, node *lntest.HarnessNode, height int32) error { var predErr error err := wait.Predicate(func() bool { - ctxt, _ := context.WithTimeout(ctx, 10*time.Second) + ctxt, _ := context.WithTimeout(ctx, defaultTimeout) info, err := node.GetInfo(ctxt, &lnrpc.GetInfoRequest{}) if err != nil { predErr = err @@ -2409,7 +2409,7 @@ func waitForNodeBlockHeight(ctx context.Context, node *lntest.HarnessNode, return false } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { return predErr } @@ -2445,7 +2445,7 @@ func assertMinerBlockHeightDelta(t *harnessTest, return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -2680,7 +2680,7 @@ func testOpenChannelAfterReorg(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -3619,7 +3619,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -3821,7 +3821,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return nil - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -3936,7 +3936,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return nil - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(err.Error()) } @@ -4069,7 +4069,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -4143,7 +4143,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return nil - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(err.Error()) } @@ -4307,7 +4307,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -4437,7 +4437,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -4475,7 +4475,7 @@ func channelForceClosureTest(net *lntest.NetworkHarness, t *harnessTest, } return true - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -5299,7 +5299,7 @@ func assertAmountPaid(t *harnessTest, channelName string, // are in place var timeover uint32 go func() { - <-time.After(time.Second * 20) + <-time.After(defaultTimeout) atomic.StoreUint32(&timeover, 1) }() @@ -6182,7 +6182,7 @@ func testUnannouncedChannels(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -6534,7 +6534,7 @@ func testPrivateChannels(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -6716,7 +6716,7 @@ func testInvoiceRoutingHints(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf(predErr.Error()) } @@ -7749,7 +7749,7 @@ func testFailingChannel(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -7785,7 +7785,7 @@ func testFailingChannel(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -7807,7 +7807,7 @@ func testFailingChannel(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -7855,7 +7855,7 @@ func testFailingChannel(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -7950,13 +7950,13 @@ func testGarbageCollectLinkNodes(net *lntest.NetworkHarness, t *harnessTest) { err = wait.Predicate(func() bool { return isConnected(net.Bob.PubKeyStr) - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("alice did not reconnect to bob") } err = wait.Predicate(func() bool { return isConnected(carol.PubKeyStr) - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("alice did not reconnect to carol") } @@ -7969,19 +7969,19 @@ func testGarbageCollectLinkNodes(net *lntest.NetworkHarness, t *harnessTest) { err = wait.Predicate(func() bool { return isConnected(net.Bob.PubKeyStr) - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("alice did not reconnect to bob") } err = wait.Predicate(func() bool { return isConnected(carol.PubKeyStr) - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("alice did not reconnect to carol") } err = wait.Predicate(func() bool { return isConnected(dave.PubKeyStr) - }, 15*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("alice did not reconnect to dave") } @@ -8019,7 +8019,7 @@ func testGarbageCollectLinkNodes(net *lntest.NetworkHarness, t *harnessTest) { } err = wait.Predicate(func() bool { return isConnected(dave.PubKeyStr) - }, 20*time.Second) + }, defaultTimeout) if err != nil { t.Fatalf("alice didn't reconnect to Dave") } @@ -8091,7 +8091,7 @@ func testGarbageCollectLinkNodes(net *lntest.NetworkHarness, t *harnessTest) { } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("channels not marked as fully resolved: %v", predErr) } @@ -8223,7 +8223,7 @@ func testRevokedCloseRetribution(net *lntest.NetworkHarness, t *harnessTest) { bobChan = bChan return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -8301,7 +8301,7 @@ func testRevokedCloseRetribution(net *lntest.NetworkHarness, t *harnessTest) { } return true - }, time.Second*10) + }, defaultTimeout) if err != nil { t.Fatalf("unable to close channel: %v", predErr) } @@ -8555,7 +8555,7 @@ func testRevokedCloseRetributionZeroValueRemoteOutput(net *lntest.NetworkHarness ctxt, carol, chanPoint, force, ) return closeErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("unable to close channel: %v", closeErr) } @@ -8966,7 +8966,7 @@ func testRevokedCloseRetributionRemoteHodl(net *lntest.NetworkHarness, justiceTxid = txid return true - }, time.Second*10) + }, defaultTimeout) if err != nil && predErr == errNotFound { // If Dave is unable to broadcast his justice tx on first // attempt because of the second layer transactions, he will @@ -8986,7 +8986,7 @@ func testRevokedCloseRetributionRemoteHodl(net *lntest.NetworkHarness, justiceTxid = txid return true - }, time.Second*10) + }, defaultTimeout) } if err != nil { t.Fatalf(predErr.Error()) @@ -9480,7 +9480,7 @@ func testRevokedCloseRetributionAltruistWatchtowerCase( } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -9504,7 +9504,7 @@ func testRevokedCloseRetributionAltruistWatchtowerCase( } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -9543,7 +9543,7 @@ func assertNumPendingChannels(t *harnessTest, node *lntest.HarnessNode, return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -9798,7 +9798,7 @@ func testDataLossProtection(net *lntest.NetworkHarness, t *harnessTest) { nodeChan = bChan return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", predErr) } @@ -10001,7 +10001,7 @@ func testDataLossProtection(net *lntest.NetworkHarness, t *harnessTest) { } return nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("%v", err) } @@ -10037,7 +10037,7 @@ func assertNodeNumChannels(t *harnessTest, node *lntest.HarnessNode, return true } - if err := wait.Predicate(pred, time.Second*15); err != nil { + if err := wait.Predicate(pred, defaultTimeout); err != nil { t.Fatalf("node has incorrect number of channels: %v", predErr) } } @@ -10583,7 +10583,7 @@ func testNodeAnnouncement(net *lntest.NetworkHarness, t *harnessTest) { } case err := <-graphSub.errChan: t.Fatalf("unable to recv graph update: %v", err) - case <-time.After(20 * time.Second): + case <-time.After(defaultTimeout): t.Fatalf("did not receive node ann update") } } @@ -11338,7 +11338,7 @@ func testSwitchCircuitPersistence(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -11373,7 +11373,7 @@ func testSwitchCircuitPersistence(net *lntest.NetworkHarness, t *harnessTest) { err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -11397,7 +11397,7 @@ func testSwitchCircuitPersistence(net *lntest.NetworkHarness, t *harnessTest) { predErr = assertNumActiveHtlcs(nodes, 0) return predErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -11655,7 +11655,7 @@ func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -11678,7 +11678,7 @@ func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { err = wait.Invariant(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil - }, time.Second*2) + }, defaultTimeout) if err != nil { t.Fatalf("htlc change: %v", predErr) } @@ -11702,7 +11702,7 @@ func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(carolNode, 0) return predErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -11721,7 +11721,7 @@ func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -11982,7 +11982,7 @@ func testSwitchOfflineDeliveryPersistence(net *lntest.NetworkHarness, t *harness } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -12019,7 +12019,7 @@ func testSwitchOfflineDeliveryPersistence(net *lntest.NetworkHarness, t *harness predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) return predErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -12049,7 +12049,7 @@ func testSwitchOfflineDeliveryPersistence(net *lntest.NetworkHarness, t *harness return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -12316,7 +12316,7 @@ func testSwitchOfflineDeliveryOutgoingOffline( return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -12344,7 +12344,7 @@ func testSwitchOfflineDeliveryOutgoingOffline( predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) return predErr == nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -12389,7 +12389,7 @@ func testSwitchOfflineDeliveryOutgoingOffline( return false } return true - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } @@ -13723,7 +13723,7 @@ func testHoldInvoicePersistence(net *lntest.NetworkHarness, t *harnessTest) { } return nil - }, time.Second*15) + }, defaultTimeout) if err != nil { t.Fatalf("predicate not satisfied: %v", err) } diff --git a/lntest/node.go b/lntest/node.go index 473bde2c..73b44acd 100644 --- a/lntest/node.go +++ b/lntest/node.go @@ -660,7 +660,7 @@ func (hn *HarnessNode) initClientWhenReady() error { if err := wait.NoError(func() error { conn, connErr = hn.ConnectRPC(true) return connErr - }, 5*time.Second); err != nil { + }, DefaultTimeout); err != nil { return err } @@ -1005,7 +1005,7 @@ func (hn *HarnessNode) stop() error { // Wait for lnd process and other goroutines to exit. select { case <-hn.processExit: - case <-time.After(60 * time.Second): + case <-time.After(DefaultTimeout * 2): return fmt.Errorf("process did not exit") } @@ -1357,7 +1357,7 @@ func (hn *HarnessNode) WaitForBalance(expectedBalance btcutil.Amount, confirmed return btcutil.Amount(balance.UnconfirmedBalance) == expectedBalance } - err := wait.Predicate(doesBalanceMatch, 30*time.Second) + err := wait.Predicate(doesBalanceMatch, DefaultTimeout) if err != nil { return fmt.Errorf("balances not synced after deadline: "+ "expected %v, only have %v", expectedBalance, lastBalance) From 809d2383642c0751c7abc8fc224f9090a0c7745d Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 16:27:03 +0100 Subject: [PATCH 5/7] itest: wait for server to start when restoring To avoid running into the "server is still starting" error when trying to close a channel, we first wait for the error to disappear before we try closing the actual channel. --- lntest/itest/lnd_channel_backup_test.go | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lntest/itest/lnd_channel_backup_test.go b/lntest/itest/lnd_channel_backup_test.go index 5c650ff4..5b6f2fcd 100644 --- a/lntest/itest/lnd_channel_backup_test.go +++ b/lntest/itest/lnd_channel_backup_test.go @@ -987,6 +987,33 @@ func testChanRestoreScenario(t *harnessTest, net *lntest.NetworkHarness, ) require.NoError(t.t, err) + // We now need to make sure the server is fully started before we can + // actually close the channel. This is the first check in CloseChannel + // so we can try with a nil channel point until we get the correct error + // to find out if Dave is fully started. + err = wait.Predicate(func() bool { + const expectedErr = "must specify channel point" + ctxc, cancel := context.WithCancel(ctxt) + defer cancel() + + resp, err := dave.CloseChannel( + ctxc, &lnrpc.CloseChannelRequest{}, + ) + if err != nil { + return false + } + + defer func() { _ = resp.CloseSend() }() + + _, err = resp.Recv() + if err != nil && strings.Contains(err.Error(), expectedErr) { + return true + } + + return false + }, defaultTimeout) + require.NoError(t.t, err) + // We also want to make sure we cannot force close in this state. That // would get the state machine in a weird state. chanPointParts := strings.Split( From a436618e4313757ecbcd932282b1f678817cc9c0 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 16:46:54 +0100 Subject: [PATCH 6/7] lntest: fix linter errors for changed code --- lntest/harness.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lntest/harness.go b/lntest/harness.go index 35391bad..bf8dd450 100644 --- a/lntest/harness.go +++ b/lntest/harness.go @@ -290,7 +290,6 @@ func (n *NetworkHarness) NewNodeWithSeed(name string, extraArgs []string, return nil, nil, nil, err } - timeout := time.Duration(DefaultTimeout) ctxb := context.Background() // Create a request to generate a new aezeed. The new seed will have the @@ -299,7 +298,8 @@ func (n *NetworkHarness) NewNodeWithSeed(name string, extraArgs []string, AezeedPassphrase: password, } - ctxt, _ := context.WithTimeout(ctxb, timeout) + ctxt, cancel := context.WithTimeout(ctxb, DefaultTimeout) + defer cancel() genSeedResp, err := node.GenSeed(ctxt, genSeedReq) if err != nil { return nil, nil, nil, err @@ -457,7 +457,8 @@ func (n *NetworkHarness) EnsureConnected(ctx context.Context, a, b *HarnessNode) errConnectionRequested := errors.New("connection request in progress") tryConnect := func(a, b *HarnessNode) error { - ctxt, _ := context.WithTimeout(ctx, DefaultTimeout) + ctxt, cancel := context.WithTimeout(ctx, DefaultTimeout) + defer cancel() bInfo, err := b.GetInfo(ctxt, &lnrpc.GetInfoRequest{}) if err != nil { return err @@ -530,7 +531,8 @@ func (n *NetworkHarness) EnsureConnected(ctx context.Context, a, b *HarnessNode) // If node B is seen in the ListPeers response from node A, // then we can exit early as the connection has been fully // established. - ctxt, _ := context.WithTimeout(ctx, DefaultTimeout) + ctxt, cancel := context.WithTimeout(ctx, DefaultTimeout) + defer cancel() resp, err := b.ListPeers(ctxt, &lnrpc.ListPeersRequest{}) if err != nil { return false From 9de659ea4f8cd7d00cbb7d97f8a264d417737275 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Tue, 8 Dec 2020 18:54:18 +0100 Subject: [PATCH 7/7] itest: use require.Eventually where tick speed matters To not waste a lot of time with the adjusted longer tick interval, we use a short interval with require.Eventually where we can save some time. --- lntest/itest/lnd_test.go | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 7cd69448..68845c9f 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -7948,18 +7948,12 @@ func testGarbageCollectLinkNodes(net *lntest.NetworkHarness, t *harnessTest) { t.Fatalf("unable to restart carol's node: %v", err) } - err = wait.Predicate(func() bool { + require.Eventually(t.t, func() bool { return isConnected(net.Bob.PubKeyStr) - }, defaultTimeout) - if err != nil { - t.Fatalf("alice did not reconnect to bob") - } - err = wait.Predicate(func() bool { + }, defaultTimeout, 20*time.Millisecond) + require.Eventually(t.t, func() bool { return isConnected(carol.PubKeyStr) - }, defaultTimeout) - if err != nil { - t.Fatalf("alice did not reconnect to carol") - } + }, defaultTimeout, 20*time.Millisecond) // We'll also restart Alice to ensure she can reconnect to her peers // with open channels. @@ -7967,24 +7961,18 @@ func testGarbageCollectLinkNodes(net *lntest.NetworkHarness, t *harnessTest) { t.Fatalf("unable to restart alice's node: %v", err) } - err = wait.Predicate(func() bool { + require.Eventually(t.t, func() bool { return isConnected(net.Bob.PubKeyStr) - }, defaultTimeout) - if err != nil { - t.Fatalf("alice did not reconnect to bob") - } - err = wait.Predicate(func() bool { + }, defaultTimeout, 20*time.Millisecond) + require.Eventually(t.t, func() bool { return isConnected(carol.PubKeyStr) - }, defaultTimeout) - if err != nil { - t.Fatalf("alice did not reconnect to carol") - } + }, defaultTimeout, 20*time.Millisecond) + require.Eventually(t.t, func() bool { + return isConnected(dave.PubKeyStr) + }, defaultTimeout, 20*time.Millisecond) err = wait.Predicate(func() bool { return isConnected(dave.PubKeyStr) }, defaultTimeout) - if err != nil { - t.Fatalf("alice did not reconnect to dave") - } // testReconnection is a helper closure that restarts the nodes at both // ends of a channel to ensure they do not reconnect after restarting.