From e6f45a948ecdeac3262199caf6f438f994cf1bc7 Mon Sep 17 00:00:00 2001 From: Andrey Samokhvalov Date: Sat, 15 Oct 2016 16:47:09 +0300 Subject: [PATCH] testing: add CT (Custom Testing) structure; create uniq point for 'lnd process errors' and 'test panic/failed errors' handling --- glide.lock | 3 + glide.yaml | 1 + lnd_test.go | 391 ++++++++++++++++++++++++++++--------------------- networktest.go | 37 ++++- 4 files changed, 264 insertions(+), 168 deletions(-) diff --git a/glide.lock b/glide.lock index 81a55faa..d593f7e7 100644 --- a/glide.lock +++ b/glide.lock @@ -155,4 +155,7 @@ imports: - naming - transport - peer +- name: github.com/go-errors/errors + version: a41850380601eeb43f4350f7d17c6bbd8944aaf8 + testImports: [] diff --git a/glide.yaml b/glide.yaml index 5ee80dad..33217684 100644 --- a/glide.yaml +++ b/glide.yaml @@ -60,3 +60,4 @@ import: - package: github.com/grpc-ecosystem/grpc-gateway version: ^1.1.0 - package: github.com/aead/chacha20 +- package: github.com/go-errors/errors diff --git a/lnd_test.go b/lnd_test.go index c0349da5..43c0a4c9 100644 --- a/lnd_test.go +++ b/lnd_test.go @@ -3,42 +3,100 @@ package main import ( "bytes" "fmt" - "runtime/debug" - "sync" - "testing" - "time" - "golang.org/x/net/context" - + "sync" + "time" "github.com/davecgh/go-spew/spew" + "github.com/go-errors/errors" "github.com/lightningnetwork/lnd/lnrpc" "github.com/roasbeef/btcd/rpctest" "github.com/roasbeef/btcd/wire" "github.com/roasbeef/btcrpcclient" "github.com/roasbeef/btcutil" + "testing" ) -func assertTxInBlock(block *btcutil.Block, txid *wire.ShaHash, t *testing.T) { +// CT is needed for: +// - have uniform way of handling panic and fatal error from test cases. +// - have ability to properly wrap errors in order to see stack trace. +// - have nice and elegant way to handle lnd process errors in one +// select structure with test cases. +type CT struct { + *testing.T + + // Channel for sending retransmitted panic errors and fatal error which + // happens in test case. + errChan chan error +} + +func NewCT(t *testing.T) *CT { + return &CT{t, nil} +} + +func (ct *CT) Error(err error) { + if ct.errChan != nil { + ct.errChan <- fmt.Errorf(errors.Wrap(err, 1).ErrorStack()) + ct.FailNow() + } else { + ct.Fatal("can't sen error when test isn't running") + } +} + +// Errorf create and send the description about the error in the error channel +// and exit. +func (ct *CT) Errorf(format string, a ...interface{}) { + if ct.errChan != nil { + description := fmt.Sprintf(format, a...) + ct.errChan <- fmt.Errorf(errors.Wrap(description, 1).ErrorStack()) + ct.FailNow() + } else { + ct.Fatal("can't sen error when test isn't running") + } + +} + +// RunTest wraps test case function in goroutine and also redirects the panic +// error from test case into error channel. +func (ct *CT) RunTest(net *networkHarness, test testCase) chan error { + // a channel to signal that test was exited with error + ct.errChan = make(chan error) + + go func() { + defer func() { + if err := recover(); err != nil { + // Retransmit test panic into main "process" + ct.errChan <- fmt.Errorf(err.(string)) + } + close(ct.errChan) + ct.errChan = nil + }() + test(net, ct) + }() + + return ct.errChan +} + +func assertTxInBlock(ct *CT, block *btcutil.Block, txid *wire.ShaHash) { for _, tx := range block.Transactions() { if bytes.Equal(txid[:], tx.Sha()[:]) { return } } - t.Fatalf("funding tx was not included in block") + ct.Errorf("funding tx was not included in block") } // openChannelAndAssert attempts to open a channel with the specified // parameters extended from Alice to Bob. Additionally, two items are asserted -// after the channel is considered open: the funding transactino should be +// after the channel is considered open: the funding transaction should be // found within a block, and that Alice can report the status of the new // channel. -func openChannelAndAssert(t *testing.T, net *networkHarness, ctx context.Context, +func openChannelAndAssert(ct *CT, net *networkHarness, ctx context.Context, alice, bob *lightningNode, amount btcutil.Amount) *lnrpc.ChannelPoint { chanOpenUpdate, err := net.OpenChannel(ctx, alice, bob, amount, 1) if err != nil { - t.Fatalf("unable to open channel: %v", err) + ct.Errorf("unable to open channel: %v", err) } // Mine a block, then wait for Alice's node to notify us that the @@ -46,21 +104,21 @@ func openChannelAndAssert(t *testing.T, net *networkHarness, ctx context.Context // within the newly mined block. blockHash, err := net.Miner.Node.Generate(1) if err != nil { - t.Fatalf("unable to generate block: %v", err) + ct.Errorf("unable to generate block: %v", err) } block, err := net.Miner.Node.GetBlock(blockHash[0]) if err != nil { - t.Fatalf("unable to get block: %v", err) + ct.Errorf("unable to get block: %v", err) } fundingChanPoint, err := net.WaitForChannelOpen(ctx, chanOpenUpdate) if err != nil { - t.Fatalf("error while waiting for channel open: %v", err) + ct.Errorf("error while waiting for channel open: %v", err) } fundingTxID, err := wire.NewShaHash(fundingChanPoint.FundingTxid) if err != nil { - t.Fatalf("unable to create sha hash: %v", err) + ct.Errorf("unable to create sha hash: %v", err) } - assertTxInBlock(block, fundingTxID, t) + assertTxInBlock(ct, block, fundingTxID) // The channel should be listed in the peer information returned by // both peers. @@ -68,27 +126,25 @@ func openChannelAndAssert(t *testing.T, net *networkHarness, ctx context.Context Hash: *fundingTxID, Index: fundingChanPoint.OutputIndex, } - err = net.AssertChannelExists(ctx, alice, &chanPoint) - if err != nil { - t.Fatalf("unable to assert channel existence: %v", err) + if err := net.AssertChannelExists(ctx, alice, &chanPoint); err != nil { + ct.Errorf("unable to assert channel existence: %v", err) } return fundingChanPoint } -// closeChannelAndAssert attemps to close a channel identified by the passed +// closeChannelAndAssert attempts to close a channel identified by the passed // channel point owned by the passed lighting node. A fully blocking channel // closure is attempted, therefore the passed context should be a child derived // via timeout from a base parent. Additionally, once the channel has been // detected as closed, an assertion checks that the transaction is found within // a block. -func closeChannelAndAssert(t *testing.T, net *networkHarness, - ctx context.Context, node *lightningNode, - fundingChanPoint *lnrpc.ChannelPoint) { +func closeChannelAndAssert(ct *CT, net *networkHarness, ctx context.Context, + node *lightningNode, fundingChanPoint *lnrpc.ChannelPoint) { closeUpdates, err := net.CloseChannel(ctx, node, fundingChanPoint, false) if err != nil { - t.Fatalf("unable to close channel: %v", err) + ct.Errorf("unable to close channel: %v", err) } // Finally, generate a single block, wait for the final close status @@ -96,19 +152,19 @@ func closeChannelAndAssert(t *testing.T, net *networkHarness, // block. blockHash, err := net.Miner.Node.Generate(1) if err != nil { - t.Fatalf("unable to generate block: %v", err) + ct.Errorf("unable to generate block: %v", err) } block, err := net.Miner.Node.GetBlock(blockHash[0]) if err != nil { - t.Fatalf("unable to get block: %v", err) + ct.Errorf("unable to get block: %v", err) } closingTxid, err := net.WaitForChannelClose(ctx, closeUpdates) if err != nil { - t.Fatalf("error while waiting for channel close: %v", err) + ct.Errorf("error while waiting for channel close: %v", err) } - assertTxInBlock(block, closingTxid, t) + assertTxInBlock(ct, block, closingTxid) } // testBasicChannelFunding performs a test exercising expected behavior from a @@ -116,7 +172,7 @@ func closeChannelAndAssert(t *testing.T, net *networkHarness, // Bob, then immediately closes the channel after asserting some expected post // conditions. Finally, the chain itself is checked to ensure the closing // transaction was mined. -func testBasicChannelFunding(net *networkHarness, t *testing.T) { +func testBasicChannelFunding(net *networkHarness, ct *CT) { timeout := time.Duration(time.Second * 5) ctxb := context.Background() @@ -128,40 +184,44 @@ func testBasicChannelFunding(net *networkHarness, t *testing.T) { // assertions will be executed to ensure the funding process completed // successfully. ctxt, _ := context.WithTimeout(ctxb, timeout) - chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, chanAmt) + chanPoint := openChannelAndAssert(ct, net, ctxt, net.Alice, net.Bob, chanAmt) // Finally, immediately close the channel. This function will also // block until the channel is closed and will additionally assert the // relevant channel closing post conditions. ctxt, _ = context.WithTimeout(ctxb, timeout) - closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint) + closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPoint) } // testChannelBalance creates a new channel between Alice and Bob, then // checks channel balance to be equal amount specified while creation of channel. -func testChannelBalance(net *networkHarness, t *testing.T) { +func testChannelBalance(net *networkHarness, ct *CT) { timeout := time.Duration(time.Second * 5) - ctxb := context.Background() - - // Creates a helper closure to be used below which asserts the proper - // response to a channel balance RPC. - checkChannelBalance := func(node lnrpc.LightningClient, amount btcutil.Amount) { - response, err := node.ChannelBalance(ctxb, &lnrpc.ChannelBalanceRequest{}) - if err != nil { - t.Fatalf("unable to get channel balance: %v", err) - } - - balance := btcutil.Amount(response.Balance) - if balance != amount { - t.Fatalf("channel balance wrong: %v != %v", balance, amount) - } - } // Open a channel with 0.5 BTC between Alice and Bob, ensuring the // channel has been opened properly. amount := btcutil.Amount(btcutil.SatoshiPerBitcoin / 2) - ctxt, _ := context.WithTimeout(ctxb, timeout) - chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, amount) + ctx, _ := context.WithTimeout(context.Background(), timeout) + + // Creates a helper closure to be used below which asserts the proper + // response to a channel balance RPC. + checkChannelBalance := func(node lnrpc.LightningClient, + amount btcutil.Amount) { + + response, err := node.ChannelBalance(ctx, &lnrpc.ChannelBalanceRequest{}) + if err != nil { + ct.Errorf("unable to get channel balance: %v", err) + } + + balance := btcutil.Amount(response.Balance) + if balance != amount { + ct.Errorf("channel balance wrong: %v != %v", balance, + amount) + } + } + + chanPoint := openChannelAndAssert(ct, net, ctx, net.Alice, net.Bob, + amount) // As this is a single funder channel, Alice's balance should be // exactly 0.5 BTC since now state transitions have taken place yet. @@ -180,8 +240,8 @@ func testChannelBalance(net *networkHarness, t *testing.T) { // Finally close the channel between Alice and Bob, asserting that the // channel has been properly closed on-chain. - ctxt, _ = context.WithTimeout(ctxb, timeout) - closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint) + ctx, _ = context.WithTimeout(context.Background(), timeout) + closeChannelAndAssert(ct, net, ctx, net.Alice, chanPoint) } // testChannelForceClosure performs a test to exercise the behavior of "force" @@ -193,7 +253,7 @@ func testChannelBalance(net *networkHarness, t *testing.T) { // once the output(s) become mature. // // TODO(roabeef): also add an unsettled HTLC before force closing. -func testChannelForceClosure(net *networkHarness, t *testing.T) { +func testChannelForceClosure(net *networkHarness, ct *CT) { timeout := time.Duration(time.Second * 5) ctxb := context.Background() @@ -204,15 +264,17 @@ func testChannelForceClosure(net *networkHarness, t *testing.T) { chanOpenUpdate, err := net.OpenChannel(ctxb, net.Alice, net.Bob, chanAmt, numFundingConfs) if err != nil { - t.Fatalf("unable to open channel: %v", err) + ct.Errorf("unable to open channel: %v", err) } + if _, err := net.Miner.Node.Generate(numFundingConfs); err != nil { - t.Fatalf("unable to mine block: %v", err) + ct.Errorf("unable to mine block: %v", err) } + ctxt, _ := context.WithTimeout(ctxb, timeout) chanPoint, err := net.WaitForChannelOpen(ctxt, chanOpenUpdate) if err != nil { - t.Fatalf("error while waiting for channel to open: %v", err) + ct.Errorf("error while waiting for channel to open: %v", err) } // Now that the channel is open, immediately execute a force closure of @@ -221,18 +283,18 @@ func testChannelForceClosure(net *networkHarness, t *testing.T) { // request. closeUpdate, err := net.CloseChannel(ctxb, net.Alice, chanPoint, true) if err != nil { - t.Fatalf("unable to execute force channel closure: %v", err) + ct.Errorf("unable to execute force channel closure: %v", err) } // Mine a block which should confirm the commitment transaction // broadcast as a result of the force closure. if _, err := net.Miner.Node.Generate(1); err != nil { - t.Fatalf("unable to generate block: %v", err) + ct.Errorf("unable to generate block: %v", err) } ctxt, _ = context.WithTimeout(ctxb, timeout) closingTxID, err := net.WaitForChannelClose(ctxt, closeUpdate) if err != nil { - t.Fatalf("error while waiting for channel close: %v", err) + ct.Errorf("error while waiting for channel close: %v", err) } // Currently within the codebase, the default CSV is 4 relative blocks. @@ -241,7 +303,7 @@ func testChannelForceClosure(net *networkHarness, t *testing.T) { // or make delay a param const defaultCSV = 4 if _, err := net.Miner.Node.Generate(defaultCSV); err != nil { - t.Fatalf("unable to mine blocks: %v", err) + ct.Errorf("unable to mine blocks: %v", err) } // At this point, the sweeping transaction should now be broadcast. So @@ -253,11 +315,11 @@ mempoolPoll: for { select { case <-time.After(time.Second * 5): - t.Fatalf("sweep tx not found in mempool") + ct.Errorf("sweep tx not found in mempool") default: mempool, err = net.Miner.Node.GetRawMempool() if err != nil { - t.Fatalf("unable to fetch node's mempool: %v", err) + ct.Errorf("unable to fetch node's mempool: %v", err) } if len(mempool) == 0 { continue @@ -271,7 +333,7 @@ mempoolPoll: // TODO(roasbeef): assertion may not necessarily hold with concurrent // test executions if len(mempool) != 1 { - t.Fatalf("node's mempool is wrong size, expected 1 got %v", + ct.Errorf("node's mempool is wrong size, expected 1 got %v", len(mempool)) } sweepingTXID = mempool[0] @@ -280,11 +342,11 @@ mempoolPoll: // the commitment transaction which was broadcast on-chain. sweepTx, err := net.Miner.Node.GetRawTransaction(sweepingTXID) if err != nil { - t.Fatalf("unable to fetch sweep tx: %v", err) + ct.Errorf("unable to fetch sweep tx: %v", err) } for _, txIn := range sweepTx.MsgTx().TxIn { if !closingTxID.IsEqual(&txIn.PreviousOutPoint.Hash) { - t.Fatalf("sweep transaction not spending from commit "+ + ct.Errorf("sweep transaction not spending from commit "+ "tx %v, instead spending %v", closingTxID, txIn.PreviousOutPoint) } @@ -295,16 +357,17 @@ mempoolPoll: // inputs should be properly met. blockHash, err := net.Miner.Node.Generate(1) if err != nil { - t.Fatalf("unable to generate block: %v", err) + ct.Errorf("unable to generate block: %v", err) } block, err := net.Miner.Node.GetBlock(blockHash[0]) if err != nil { - t.Fatalf("unable to get block: %v", err) + ct.Errorf("unable to get block: %v", err) } - assertTxInBlock(block, sweepTx.Sha(), t) + + assertTxInBlock(ct, block, sweepTx.Sha()) } -func testSingleHopInvoice(net *networkHarness, t *testing.T) { +func testSingleHopInvoice(net *networkHarness, ct *CT) { ctxb := context.Background() timeout := time.Duration(time.Second * 5) @@ -312,7 +375,7 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) { // the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, timeout) chanAmt := btcutil.Amount(100000) - chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, chanAmt) + chanPoint := openChannelAndAssert(ct, net, ctxt, net.Alice, net.Bob, chanAmt) // Now that the channel is open, create an invoice for Bob which // expects a payment of 1000 satoshis from Alice paid via a particular @@ -326,14 +389,14 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) { } invoiceResp, err := net.Bob.AddInvoice(ctxb, invoice) if err != nil { - t.Fatalf("unable to add invoice: %v", err) + ct.Errorf("unable to add invoice: %v", err) } // With the invoice for Bob added, send a payment towards Alice paying // to the above generated invoice. sendStream, err := net.Alice.SendPayment(ctxb) if err != nil { - t.Fatalf("unable to create alice payment stream: %v", err) + ct.Errorf("unable to create alice payment stream: %v", err) } sendReq := &lnrpc.SendRequest{ PaymentHash: invoiceResp.RHash, @@ -341,10 +404,10 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) { Amt: paymentAmt, } if err := sendStream.Send(sendReq); err != nil { - t.Fatalf("unable to send payment: %v", err) + ct.Errorf("unable to send payment: %v", err) } if _, err := sendStream.Recv(); err != nil { - t.Fatalf("error when attempting recv: %v", err) + ct.Errorf("error when attempting recv: %v", err) } // Bob's invoice should now be found and marked as settled. @@ -355,37 +418,38 @@ func testSingleHopInvoice(net *networkHarness, t *testing.T) { } dbInvoice, err := net.Bob.LookupInvoice(ctxb, payHash) if err != nil { - t.Fatalf("unable to lookup invoice: %v", err) + ct.Errorf("unable to lookup invoice: %v", err) } if !dbInvoice.Settled { - t.Fatalf("bob's invoice should be marked as settled: %v", + ct.Errorf("bob's invoice should be marked as settled: %v", spew.Sdump(dbInvoice)) } // The balances of Alice and Bob should be updated accordingly. aliceBalance, err := net.Alice.ChannelBalance(ctxb, &lnrpc.ChannelBalanceRequest{}) if err != nil { - t.Fatalf("unable to query for alice's balance: %v", err) + ct.Errorf("unable to query for alice's balance: %v", err) } bobBalance, err := net.Bob.ChannelBalance(ctxb, &lnrpc.ChannelBalanceRequest{}) if err != nil { - t.Fatalf("unable to query for bob's balance: %v", err) + ct.Errorf("unable to query for bob's balance: %v", err) } if aliceBalance.Balance != int64(chanAmt-paymentAmt) { - t.Fatalf("Alice's balance is incorrect got %v, expected %v", + ct.Errorf("Alice's balance is incorrect got %v, expected %v", aliceBalance, int64(chanAmt-paymentAmt)) } if bobBalance.Balance != paymentAmt { - t.Fatalf("Bob's balance is incorrect got %v, expected %v", + ct.Errorf("Bob's balance is incorrect got %v, expected %v", bobBalance, paymentAmt) } ctxt, _ = context.WithTimeout(ctxb, timeout) - closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint) + closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPoint) } -func testMultiHopPayments(net *networkHarness, t *testing.T) { +func testMultiHopPayments(net *networkHarness, ct *CT) { + const chanAmt = btcutil.Amount(100000) ctxb := context.Background() timeout := time.Duration(time.Second * 5) @@ -393,11 +457,12 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { // Open a channel with 100k satoshis between Alice and Bob with Alice // being the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, timeout) - chanPointAlice := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, - chanAmt) + chanPointAlice := openChannelAndAssert(ct, net, ctxt, net.Alice, + net.Bob, chanAmt) + aliceChanTXID, err := wire.NewShaHash(chanPointAlice.FundingTxid) if err != nil { - t.Fatalf("unable to create sha hash: %v", err) + ct.Errorf("unable to create sha hash: %v", err) } aliceFundPoint := wire.OutPoint{ Hash: *aliceChanTXID, @@ -411,21 +476,22 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { // The network topology should now look like: Carol -> Alice -> Bob carol, err := net.NewNode(nil) if err != nil { - t.Fatalf("unable to create new nodes: %v", err) + ct.Errorf("unable to create new nodes: %v", err) } if err := net.ConnectNodes(ctxb, carol, net.Alice); err != nil { - t.Fatalf("unable to connect carol to alice: %v", err) + ct.Errorf("unable to connect carol to alice: %v", err) } err = net.SendCoins(ctxb, btcutil.SatoshiPerBitcoin, carol) if err != nil { - t.Fatalf("unable to send coins to carol: %v", err) + ct.Errorf("unable to send coins to carol: %v", err) } ctxt, _ = context.WithTimeout(ctxb, timeout) - chanPointCarol := openChannelAndAssert(t, net, ctxt, carol, net.Alice, - chanAmt) + chanPointCarol := openChannelAndAssert(ct, net, ctxt, carol, + net.Alice, chanAmt) + carolChanTXID, err := wire.NewShaHash(chanPointCarol.FundingTxid) if err != nil { - t.Fatalf("unable to create sha hash: %v", err) + ct.Errorf("unable to create sha hash: %v", err) } carolFundPoint := wire.OutPoint{ Hash: *carolChanTXID, @@ -446,7 +512,7 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { } resp, err := net.Bob.AddInvoice(ctxb, invoice) if err != nil { - t.Fatalf("unable to add invoice: %v", err) + ct.Errorf("unable to add invoice: %v", err) } rHashes[i] = resp.RHash @@ -459,10 +525,10 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { req := &lnrpc.ShowRoutingTableRequest{} routingResp, err := carol.ShowRoutingTable(ctxb, req) if err != nil { - t.Fatalf("unable to query for carol's routing table: %v", err) + ct.Errorf("unable to query for carol's routing table: %v", err) } if len(routingResp.Channels) != 2 { - t.Fatalf("only two channels should be seen as active in the "+ + ct.Errorf("only two channels should be seen as active in the "+ "network, instead %v are", len(routingResp.Channels)) } for _, link := range routingResp.Channels { @@ -476,7 +542,7 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { link.Id2 == net.Alice.PubKeyStr: continue default: - t.Fatalf("unkown link within routing "+ + ct.Errorf("unkown link within routing "+ "table: %v", spew.Sdump(link)) } case link.Outpoint == carolFundPoint.String(): @@ -488,11 +554,11 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { link.Id2 == net.Alice.PubKeyStr: continue default: - t.Fatalf("unkown link within routing "+ + ct.Errorf("unkown link within routing "+ "table: %v", spew.Sdump(link)) } default: - t.Fatalf("unkown channel %v found in routing table, "+ + ct.Errorf("unkown channel %v found in routing table, "+ "only %v and %v should exist", link.Outpoint, aliceFundPoint, carolFundPoint) } @@ -501,7 +567,7 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { // Using Carol as the source, pay to the 5 invoices from Bob created above. carolPayStream, err := carol.SendPayment(ctxb) if err != nil { - t.Fatalf("unable to create payment stream for carol: %v", err) + ct.Errorf("unable to create payment stream for carol: %v", err) } // Concurrently pay off all 5 of Bob's invoices. Each of the goroutines @@ -518,10 +584,10 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { wg.Add(1) go func() { if err := carolPayStream.Send(sendReq); err != nil { - t.Fatalf("unable to send payment: %v", err) + ct.Errorf("unable to send payment: %v", err) } if _, err := carolPayStream.Recv(); err != nil { - t.Fatalf("unable to recv pay resp: %v", err) + ct.Errorf("unable to recv pay resp: %v", err) } wg.Done() }() @@ -535,16 +601,18 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { select { case <-time.After(time.Second * 10): - t.Fatalf("HLTC's not cleared after 10 seconds") + ct.Errorf("HLTC's not cleared after 10 seconds") case <-finClear: } assertAsymmetricBalance := func(node *lightningNode, - chanPoint *wire.OutPoint, localBalance, remoteBalance int64) { + chanPoint *wire.OutPoint, localBalance, + remoteBalance int64) { + listReq := &lnrpc.ListChannelsRequest{} resp, err := node.ListChannels(ctxb, listReq) if err != nil { - t.Fatalf("unable to for node's channels: %v", err) + ct.Errorf("unable to for node's channels: %v", err) } for _, channel := range resp.Channels { if channel.ChannelPoint != chanPoint.String() { @@ -553,12 +621,12 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { if channel.LocalBalance != localBalance || channel.RemoteBalance != remoteBalance { - t.Fatalf("incorrect balances: %v", + ct.Errorf("incorrect balances: %v", spew.Sdump(channel)) } return } - t.Fatalf("channel not found") + ct.Errorf("channel not found") } // At this point all the channels within our proto network should be @@ -575,12 +643,12 @@ func testMultiHopPayments(net *networkHarness, t *testing.T) { assertAsymmetricBalance(net.Bob, &aliceFundPoint, sinkBal, sourceBal) ctxt, _ = context.WithTimeout(ctxb, timeout) - closeChannelAndAssert(t, net, ctxt, net.Alice, chanPointAlice) + closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPointAlice) ctxt, _ = context.WithTimeout(ctxb, timeout) - closeChannelAndAssert(t, net, ctxt, carol, chanPointCarol) + closeChannelAndAssert(ct, net, ctxt, carol, chanPointCarol) } -func testInvoiceSubscriptions(net *networkHarness, t *testing.T) { +func testInvoiceSubscriptions(net *networkHarness, ct *CT) { const chanAmt = btcutil.Amount(500000) ctxb := context.Background() timeout := time.Duration(time.Second * 5) @@ -588,7 +656,7 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) { // Open a channel with 500k satoshis between Alice and Bob with Alice // being the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, timeout) - chanPoint := openChannelAndAssert(t, net, ctxt, net.Alice, net.Bob, + chanPoint := openChannelAndAssert(ct, net, ctxt, net.Alice, net.Bob, chanAmt) // Next create a new invoice for Bob requesting 1k satoshis. @@ -601,7 +669,7 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) { } invoiceResp, err := net.Bob.AddInvoice(ctxb, invoice) if err != nil { - t.Fatalf("unable to add invoice: %v", err) + ct.Errorf("unable to add invoice: %v", err) } // Create a new invoice subscription client for Bob, the notification @@ -609,23 +677,23 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) { req := &lnrpc.InvoiceSubscription{} bobInvoiceSubscription, err := net.Bob.SubscribeInvoices(ctxb, req) if err != nil { - t.Fatalf("unable to subscribe to bob's invoice updates: %v", err) + ct.Errorf("unable to subscribe to bob's invoice updates: %v", err) } updateSent := make(chan struct{}) go func() { invoiceUpdate, err := bobInvoiceSubscription.Recv() if err != nil { - t.Fatalf("unable to recv invoice update: %v", err) + ct.Errorf("unable to recv invoice update: %v", err) } // The invoice update should exactly match the invoice created // above, but should now be settled. if !invoiceUpdate.Settled { - t.Fatalf("invoice not settled but shoudl be") + ct.Errorf("invoice not settled but shoudl be") } if !bytes.Equal(invoiceUpdate.RPreimage, invoice.RPreimage) { - t.Fatalf("payment preimages don't match: expected %v, got %v", + ct.Errorf("payment preimages don't match: expected %v, got %v", invoice.RPreimage, invoiceUpdate.RPreimage) } @@ -636,7 +704,7 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) { // which should finalize and settle the invoice. sendStream, err := net.Alice.SendPayment(ctxb) if err != nil { - t.Fatalf("unable to create alice payment stream: %v", err) + ct.Errorf("unable to create alice payment stream: %v", err) } sendReq := &lnrpc.SendRequest{ PaymentHash: invoiceResp.RHash, @@ -644,24 +712,24 @@ func testInvoiceSubscriptions(net *networkHarness, t *testing.T) { Amt: paymentAmt, } if err := sendStream.Send(sendReq); err != nil { - t.Fatalf("unable to send payment: %v", err) + ct.Errorf("unable to send payment: %v", err) } if _, err := sendStream.Recv(); err != nil { - t.Fatalf("error when attempting recv: %v", err) + ct.Errorf("error when attempting recv: %v", err) } select { case <-time.After(time.Second * 5): - t.Fatalf("update not sent after 5 seconds") + ct.Errorf("update not sent after 5 seconds") case <-updateSent: // Fall through on success } ctxt, _ = context.WithTimeout(ctxb, timeout) - closeChannelAndAssert(t, net, ctxt, net.Alice, chanPoint) + closeChannelAndAssert(ct, net, ctxt, net.Alice, chanPoint) } // testBasicChannelCreation test multiple channel opening and closing. -func testBasicChannelCreation(net *networkHarness, t *testing.T) { +func testBasicChannelCreation(net *networkHarness, ct *CT) { timeout := time.Duration(time.Second * 5) ctx, _ := context.WithTimeout(context.Background(), timeout) @@ -673,20 +741,20 @@ func testBasicChannelCreation(net *networkHarness, t *testing.T) { // channel has been properly open on-chain. chanPoints := make([]*lnrpc.ChannelPoint, num) for i := 0; i < num; i++ { - chanPoints[i] = openChannelAndAssert(t, net, ctx, net.Alice, + chanPoints[i] = openChannelAndAssert(ct, net, ctx, net.Alice, net.Bob, amount) } // Close the channel between Alice and Bob, asserting that the // channel has been properly closed on-chain. for _, chanPoint := range chanPoints { - closeChannelAndAssert(t, net, ctx, net.Alice, chanPoint) + closeChannelAndAssert(ct, net, ctx, net.Alice, chanPoint) } } -type lndTestCase func(net *networkHarness, t *testing.T) +type testCase func(net *networkHarness, t *testing.T) -var lndTestCases = map[string]lndTestCase{ +var testCases = map[string]testCase{ "basic funding flow": testBasicChannelFunding, "channel force closure": testChannelForceClosure, "channel balance": testChannelBalance, @@ -699,72 +767,69 @@ var lndTestCases = map[string]lndTestCase{ // TestLightningNetworkDaemon performs a series of integration tests amongst a // programmatically driven network of lnd nodes. func TestLightningNetworkDaemon(t *testing.T) { - var ( - btcdHarness *rpctest.Harness - lightningNetwork *networkHarness - currentTest string - err error - ) - - defer func() { - // If one of the integration tests caused a panic within the main - // goroutine, then tear down all the harnesses in order to avoid - // any leaked processes. - if r := recover(); r != nil { - fmt.Println("recovering from test panic: ", r) - if err := btcdHarness.TearDown(); err != nil { - fmt.Println("unable to tear btcd harnesses: ", err) - } - if err := lightningNetwork.TearDownAll(); err != nil { - fmt.Println("unable to tear lnd harnesses: ", err) - } - t.Fatalf("test %v panicked: %s", currentTest, debug.Stack()) - } - }() + ct := NewCT(t) // First create the network harness to gain access to its // 'OnTxAccepted' call back. - lightningNetwork, err = newNetworkHarness() + lndHarness, err := newNetworkHarness() if err != nil { - t.Fatalf("unable to create lightning network harness: %v", err) + ct.Fatalf("unable to create lightning network harness: %v", err) } - defer lightningNetwork.TearDownAll() + defer lndHarness.TearDownAll() handlers := &btcrpcclient.NotificationHandlers{ - OnTxAccepted: lightningNetwork.OnTxAccepted, + OnTxAccepted: lndHarness.OnTxAccepted, } // First create an instance of the btcd's rpctest.Harness. This will be // used to fund the wallets of the nodes within the test network and to // drive blockchain related events within the network. - btcdHarness, err = rpctest.New(harnessNetParams, handlers, nil) + btcdHarness, err := rpctest.New(harnessNetParams, handlers, nil) if err != nil { - t.Fatalf("unable to create mining node: %v", err) + ct.Fatalf("unable to create mining node: %v", err) } defer btcdHarness.TearDown() - if err = btcdHarness.SetUp(true, 50); err != nil { - t.Fatalf("unable to set up mining node: %v", err) + if err := btcdHarness.SetUp(true, 50); err != nil { + ct.Fatalf("unable to set up mining node: %v", err) } if err := btcdHarness.Node.NotifyNewTransactions(false); err != nil { - t.Fatalf("unable to request transaction notifications: %v", err) + ct.Fatalf("unable to request transaction notifications: %v", err) } // With the btcd harness created, we can now complete the // initialization of the network. args - list of lnd arguments, // example: "--debuglevel=debug" // TODO(roasbeef): create master balanced channel with all the monies? - if err := lightningNetwork.InitializeSeedNodes(btcdHarness, nil); err != nil { - t.Fatalf("unable to initialize seed nodes: %v", err) + if err := lndHarness.InitializeSeedNodes(btcdHarness, nil); err != nil { + ct.Fatalf("unable to initialize seed nodes: %v", err) } - if err = lightningNetwork.SetUp(); err != nil { - t.Fatalf("unable to set up test lightning network: %v", err) + if err = lndHarness.SetUp(); err != nil { + ct.Fatalf("unable to set up test lightning network: %v", err) } - t.Logf("Running %v integration tests", len(lndTestCases)) - for testName, lnTest := range lndTestCases { - t.Logf("Executing test %v", testName) + ct.Logf("Running %v integration tests", len(testCases)) - currentTest = testName - lnTest(lightningNetwork, t) + for name, test := range testCases { + errChan := ct.RunTest(lndHarness, test) + + select { + // Receive both types of err - panic and fatal from + // one channel and raise the fatal in main goroutine. + case err := <-errChan: + if err != nil { + ct.Fatalf("Fail: (%v): exited with error: \n%v", + name, err) + } + ct.Logf("Successed: (%v)", name) + + // In this case lightning node process finished with error + // status. It might be because of wrong flag, or it might + // be because of nil pointer access. Who knows!? Who knows... + // TODO(andrew.shvv) When two nodes are closing simultanisly + // it leads to panic - 'sending to the closed channel'. Fix it? + case err := <-lndHarness.lndErrorChan: + ct.Fatalf("Fail: (%v): lnd finished with error "+ + "(stderr): \n%v", name, err) + } } } diff --git a/networktest.go b/networktest.go index c8787968..0a17054d 100644 --- a/networktest.go +++ b/networktest.go @@ -19,6 +19,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/grpclog" + "bytes" + "github.com/go-errors/errors" "github.com/lightningnetwork/lnd/lnrpc" "github.com/roasbeef/btcd/chaincfg" "github.com/roasbeef/btcd/rpctest" @@ -159,14 +161,27 @@ func (l *lightningNode) genArgs() []string { // start launches a new process running lnd. Additionally, the PID of the // launched process is saved in order to possibly kill the process forcibly // later. -func (l *lightningNode) start() error { +func (l *lightningNode) start(lndError chan error) error { args := l.genArgs() l.cmd = exec.Command("lnd", args...) + + // Redirect stderr output to buffer + var errb bytes.Buffer + l.cmd.Stderr = &errb + if err := l.cmd.Start(); err != nil { return err } + go func() { + // If lightning node process exited with error status + // then we should transmit stderr output in main process. + if err := l.cmd.Wait(); err != nil { + lndError <- errors.New(errb.String()) + } + }() + pid, err := os.Create(filepath.Join(l.cfg.DataDir, fmt.Sprintf("%s.pid", l.nodeId))) if err != nil { @@ -234,7 +249,14 @@ func (l *lightningNode) cleanup() error { // stop attempts to stop the active lnd process. func (l *lightningNode) stop() error { - if l.cmd == nil || l.cmd.Process == nil { + + // We should skip node stop in case: + // - start of the node wasn't initiated + // - process wasn't spawned + // - process already finished + if l.cmd == nil || + l.cmd.Process == nil || + (l.cmd.ProcessState != nil && l.cmd.ProcessState.Exited()) { return nil } @@ -276,6 +298,10 @@ type networkHarness struct { seenTxns chan wire.ShaHash watchRequests chan *watchRequest + // Channel for transmitting stderr output from failed lightning node + // to main process. + lndErrorChan chan error + sync.Mutex } @@ -288,6 +314,7 @@ func newNetworkHarness() (*networkHarness, error) { activeNodes: make(map[int]*lightningNode), seenTxns: make(chan wire.ShaHash), watchRequests: make(chan *watchRequest), + lndErrorChan: make(chan error), }, nil } @@ -345,7 +372,7 @@ func (n *networkHarness) SetUp() error { go func() { var err error defer wg.Done() - if err = n.Alice.start(); err != nil { + if err = n.Alice.start(n.lndErrorChan); err != nil { errChan <- err return } @@ -353,7 +380,7 @@ func (n *networkHarness) SetUp() error { go func() { var err error defer wg.Done() - if err = n.Bob.start(); err != nil { + if err = n.Bob.start(n.lndErrorChan); err != nil { errChan <- err return } @@ -462,7 +489,7 @@ func (n *networkHarness) NewNode(extraArgs []string) (*lightningNode, error) { return nil, err } - if err := node.start(); err != nil { + if err := node.start(n.lndErrorChan); err != nil { return nil, err }