lnd: Improve error handling for node failures in integration tests.

If an lnd node encounters a fatal process errors, we now log the error
asynchronously, instead of calling Fatalf on the test instance. This
is because calling Fatalf in a separate goroutine does not actually
end the test immediately as expected, but rather just kills the
goroutine. This ensures that we see server errors on all nodes before
the test process exits.
This commit is contained in:
Jim Posen 2017-10-24 19:35:54 -07:00 committed by Olaoluwa Osuntokun
parent 4e615f9afb
commit 643c23f978
2 changed files with 16 additions and 17 deletions

@ -4049,17 +4049,17 @@ func TestLightningNetworkDaemon(t *testing.T) {
// Spawn a new goroutine to watch for any fatal errors that any of the // Spawn a new goroutine to watch for any fatal errors that any of the
// running lnd processes encounter. If an error occurs, then the test // running lnd processes encounter. If an error occurs, then the test
// fails immediately with a fatal error, as far as fatal is happening // case should naturally as a result and we log the server error here to
// inside goroutine main goroutine would not be finished at the same // help debug.
// time as we receive fatal error from lnd process.
testsFin := make(chan struct{})
go func() { go func() {
select { for {
case err := <-lndHarness.ProcessErrors(): select {
ht.Fatalf("lnd finished with error (stderr): "+ case err, more := <-lndHarness.ProcessErrors():
"\n%v", err) if !more {
case <-testsFin: return
return }
ht.Logf("lnd finished with error (stderr):\n%v", err)
}
} }
}() }()
@ -4118,6 +4118,4 @@ func TestLightningNetworkDaemon(t *testing.T) {
break break
} }
} }
close(testsFin)
} }

@ -235,13 +235,12 @@ func (l *lightningNode) Start(lndError chan error) error {
// process errors to the goroutine running the tests. // process errors to the goroutine running the tests.
go func() { go func() {
err := l.cmd.Wait() err := l.cmd.Wait()
if err != nil {
lndError <- errors.Errorf("%v\n%v\n", err, errb.String())
}
// Signal any onlookers that this process has exited. // Signal any onlookers that this process has exited.
close(l.processExit) close(l.processExit)
if err != nil {
lndError <- errors.New(errb.String())
}
}() }()
pid, err := os.Create(filepath.Join(l.cfg.DataDir, pid, err := os.Create(filepath.Join(l.cfg.DataDir,
@ -728,7 +727,7 @@ func (n *networkHarness) InitializeSeedNodes(r *rpctest.Harness, lndArgs []strin
// ProcessErrors returns a channel used for reporting any fatal process errors. // ProcessErrors returns a channel used for reporting any fatal process errors.
// If any of the active nodes within the harness' test network incur a fatal // If any of the active nodes within the harness' test network incur a fatal
// error, that error is sent over this channel. // error, that error is sent over this channel.
func (n *networkHarness) ProcessErrors() chan error { func (n *networkHarness) ProcessErrors() <-chan error {
return n.lndErrorChan return n.lndErrorChan
} }
@ -751,6 +750,7 @@ func (n *networkHarness) SetUp() error {
// Swap out grpc's default logger with out fake logger which drops the // Swap out grpc's default logger with out fake logger which drops the
// statements on the floor. // statements on the floor.
grpclog.SetLogger(&fakeLogger{}) grpclog.SetLogger(&fakeLogger{})
// Start the initial seeder nodes within the test network, then connect // Start the initial seeder nodes within the test network, then connect
// their respective RPC clients. // their respective RPC clients.
var wg sync.WaitGroup var wg sync.WaitGroup
@ -864,6 +864,7 @@ func (n *networkHarness) TearDownAll() error {
} }
} }
close(n.lndErrorChan)
return nil return nil
} }