routing: account for payment terminal errors

If we have processed a terminal state while we're pathfinding
for another shard, the payment loop should not error out on
ErrPaymentTerminal. Instead, it would wait for our shards to
complete then cleanly exit.

routing/payment_lifecycle.go

@@ -247,7 +247,18 @@ func (p *paymentLifecycle) resumePayment() ([32]byte, *route.Route, error) {
 
 		// We found a route to try, launch a new shard.
 		attempt, outcome, err := shardHandler.launchShard(rt)
-		if err != nil {
+		switch {
+		// We may get a terminal error if we've processed a shard with
+		// a terminal state (settled or permanent failure), while we
+		// were pathfinding. We know we're in a terminal state here,
+		// so we can continue and wait for our last shards to return.
+		case err == channeldb.ErrPaymentTerminal:
+			log.Infof("Payment: %v in terminal state, abandoning "+
+				"shard", p.paymentHash)
+
+			continue
+
+		case err != nil:
 			return [32]byte{}, nil, err
 		}
 

routing/payment_lifecycle_test.go

@@ -618,10 +618,8 @@ func TestRouterPaymentStateMachine(t *testing.T) {
 		{
 			// A MP payment scenario when our path finding returns
 			// after we've just received a terminal failure, and
-			// demonstrates a bug where the payment will return with
+			// attempts to dispatch a new shard. Testing that we
-			// and "unexpected" ErrPaymentTerminal rather than
+			// correctly abandon the shard and conclude the payment.
-			// failing with a permanent error. This results in the
-			// payment getting stuck.
 			name: "MP path found after failure",
 
 			steps: []string{
@@ -648,7 +646,7 @@ func TestRouterPaymentStateMachine(t *testing.T) {
 			routes: []*route.Route{
 				shard, shard,
 			},
-			paymentErr: channeldb.ErrPaymentTerminal,
+			paymentErr: channeldb.FailureReasonPaymentDetails,
 		},
 		{
 			// A MP payment scenario when our path finding returns
@@ -698,7 +696,7 @@ func TestRouterPaymentStateMachine(t *testing.T) {
 			routes: []*route.Route{
 				shard, shard, shard, shard,
 			},
-			paymentErr: channeldb.ErrPaymentTerminal,
+			paymentErr: channeldb.FailureReasonPaymentDetails,
 		},
 	}