From 643c23f9783d689e8acc1f999b0b1416b876930a Mon Sep 17 00:00:00 2001
From: Jim Posen <jimpo@coinbase.com>
Date: Tue, 24 Oct 2017 19:35:54 -0700
Subject: [PATCH] lnd: Improve error handling for node failures in integration
 tests.

If an lnd node encounters a fatal process errors, we now log the error
asynchronously, instead of calling Fatalf on the test instance. This
is because calling Fatalf in a separate goroutine does not actually
end the test immediately as expected, but rather just kills the
goroutine. This ensures that we see server errors on all nodes before
the test process exits.
---
 lnd_test.go    | 22 ++++++++++------------
 networktest.go | 11 ++++++-----
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/lnd_test.go b/lnd_test.go
index 14690a43..1e240ffd 100644
--- a/lnd_test.go
+++ b/lnd_test.go
@@ -4049,17 +4049,17 @@ func TestLightningNetworkDaemon(t *testing.T) {
 
 	// Spawn a new goroutine to watch for any fatal errors that any of the
 	// running lnd processes encounter. If an error occurs, then the test
-	// fails immediately with a fatal error, as far as fatal is happening
-	// inside goroutine main goroutine would not be finished at the same
-	// time as we receive fatal error from lnd process.
-	testsFin := make(chan struct{})
+	// case should naturally as a result and we log the server error here to
+	// help debug.
 	go func() {
-		select {
-		case err := <-lndHarness.ProcessErrors():
-			ht.Fatalf("lnd finished with error (stderr): "+
-				"\n%v", err)
-		case <-testsFin:
-			return
+		for {
+			select {
+			case err, more := <-lndHarness.ProcessErrors():
+				if !more {
+					return
+				}
+				ht.Logf("lnd finished with error (stderr):\n%v", err)
+			}
 		}
 	}()
 
@@ -4118,6 +4118,4 @@ func TestLightningNetworkDaemon(t *testing.T) {
 			break
 		}
 	}
-
-	close(testsFin)
 }
diff --git a/networktest.go b/networktest.go
index 03b51b76..8306c2d7 100644
--- a/networktest.go
+++ b/networktest.go
@@ -235,13 +235,12 @@ func (l *lightningNode) Start(lndError chan error) error {
 	// process errors to the goroutine running the tests.
 	go func() {
 		err := l.cmd.Wait()
+		if err != nil {
+			lndError <- errors.Errorf("%v\n%v\n", err, errb.String())
+		}
 
 		// Signal any onlookers that this process has exited.
 		close(l.processExit)
-
-		if err != nil {
-			lndError <- errors.New(errb.String())
-		}
 	}()
 
 	pid, err := os.Create(filepath.Join(l.cfg.DataDir,
@@ -728,7 +727,7 @@ func (n *networkHarness) InitializeSeedNodes(r *rpctest.Harness, lndArgs []strin
 // ProcessErrors returns a channel used for reporting any fatal process errors.
 // If any of the active nodes within the harness' test network incur a fatal
 // error, that error is sent over this channel.
-func (n *networkHarness) ProcessErrors() chan error {
+func (n *networkHarness) ProcessErrors() <-chan error {
 	return n.lndErrorChan
 }
 
@@ -751,6 +750,7 @@ func (n *networkHarness) SetUp() error {
 	// Swap out grpc's default logger with out fake logger which drops the
 	// statements on the floor.
 	grpclog.SetLogger(&fakeLogger{})
+
 	// Start the initial seeder nodes within the test network, then connect
 	// their respective RPC clients.
 	var wg sync.WaitGroup
@@ -864,6 +864,7 @@ func (n *networkHarness) TearDownAll() error {
 		}
 	}
 
+	close(n.lndErrorChan)
 	return nil
 }