lntest: retry node shutdown attempts to recovery tests

In #5364 we added a new error path in the `StopDaemon` method to return
an error if shutdown was attempted while a rescan/recover instance was
in progress. Since the wallet actually won't fully stop (atm)
mid-recovery, the call effectively didn't do anything in that scenario,
so we started to return an error to properly reflect that. However this
causes certain itests to fail, as during recovery, the stop attempt will
fail leading to the test itself failing.

In this commit, we wrap the calls to stop a running daemon within a
`wait.NoError` call so we'll continually try to shut down the daemon
rather than quit on the first try.

Fixes #5423.

lntest/itest/lnd_test.go

@@ -594,7 +594,13 @@ func assertNumConnections(t *harnessTest, alice, bob *lntest.HarnessNode,
 // occur.
 func shutdownAndAssert(net *lntest.NetworkHarness, t *harnessTest,
 	node *lntest.HarnessNode) {
-	if err := net.ShutdownNode(node); err != nil {
+
+	// The process may not be in a state to always shutdown immediately, so
+	// we'll retry up to a hard limit to ensure we eventually shutdown.
+	err := wait.NoError(func() error {
+		return net.ShutdownNode(node)
+	}, defaultTimeout)
+	if err != nil {
 		t.Fatalf("unable to shutdown %v: %v", node.Name(), err)
 	}
 }

lntest/node.go

@@ -1115,7 +1115,10 @@ func (hn *HarnessNode) stop() error {
 		// closed before a response is returned.
 		req := lnrpc.StopRequest{}
 		ctx := context.Background()
-		hn.LightningClient.StopDaemon(ctx, &req)
+		_, err := hn.LightningClient.StopDaemon(ctx, &req)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Wait for lnd process and other goroutines to exit.