lnd: Improve error handling for node failures in integration tests.

If an lnd node encounters a fatal process errors, we now log the error
asynchronously, instead of calling Fatalf on the test instance. This
is because calling Fatalf in a separate goroutine does not actually
end the test immediately as expected, but rather just kills the
goroutine. This ensures that we see server errors on all nodes before
the test process exits.

lnd_test.go

@@ -4049,17 +4049,17 @@ func TestLightningNetworkDaemon(t *testing.T) {
 
 	// Spawn a new goroutine to watch for any fatal errors that any of the
 	// running lnd processes encounter. If an error occurs, then the test
-	// fails immediately with a fatal error, as far as fatal is happening
-	// inside goroutine main goroutine would not be finished at the same
-	// time as we receive fatal error from lnd process.
-	testsFin := make(chan struct{})
+	// case should naturally as a result and we log the server error here to
+	// help debug.
 	go func() {
-		select {
-		case err := <-lndHarness.ProcessErrors():
-			ht.Fatalf("lnd finished with error (stderr): "+
-				"\n%v", err)
-		case <-testsFin:
-			return
+		for {
+			select {
+			case err, more := <-lndHarness.ProcessErrors():
+				if !more {
+					return
+				}
+				ht.Logf("lnd finished with error (stderr):\n%v", err)
+			}
 		}
 	}()
 
@@ -4118,6 +4118,4 @@ func TestLightningNetworkDaemon(t *testing.T) {
 			break
 		}
 	}
-
-	close(testsFin)
 }

networktest.go

@@ -235,13 +235,12 @@ func (l *lightningNode) Start(lndError chan error) error {
 	// process errors to the goroutine running the tests.
 	go func() {
 		err := l.cmd.Wait()
+		if err != nil {
+			lndError <- errors.Errorf("%v\n%v\n", err, errb.String())
+		}
 
 		// Signal any onlookers that this process has exited.
 		close(l.processExit)
-
-		if err != nil {
-			lndError <- errors.New(errb.String())
-		}
 	}()
 
 	pid, err := os.Create(filepath.Join(l.cfg.DataDir,
@@ -728,7 +727,7 @@ func (n *networkHarness) InitializeSeedNodes(r *rpctest.Harness, lndArgs []strin
 // ProcessErrors returns a channel used for reporting any fatal process errors.
 // If any of the active nodes within the harness' test network incur a fatal
 // error, that error is sent over this channel.
-func (n *networkHarness) ProcessErrors() chan error {
+func (n *networkHarness) ProcessErrors() <-chan error {
 	return n.lndErrorChan
 }
 
@@ -751,6 +750,7 @@ func (n *networkHarness) SetUp() error {
 	// Swap out grpc's default logger with out fake logger which drops the
 	// statements on the floor.
 	grpclog.SetLogger(&fakeLogger{})
+
 	// Start the initial seeder nodes within the test network, then connect
 	// their respective RPC clients.
 	var wg sync.WaitGroup
@@ -864,6 +864,7 @@ func (n *networkHarness) TearDownAll() error {
 		}
 	}
 
+	close(n.lndErrorChan)
 	return nil
 }