diff --git a/lntest/itest/lnd_switch_test.go b/lntest/itest/lnd_switch_test.go new file mode 100644 index 00000000..81541110 --- /dev/null +++ b/lntest/itest/lnd_switch_test.go @@ -0,0 +1,1195 @@ +package itest + +import ( + "context" + "time" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lntest" + "github.com/lightningnetwork/lnd/lntest/wait" +) + +// testSwitchCircuitPersistence creates a multihop network to ensure the sender +// and intermediaries are persisting their open payment circuits. After +// forwarding a packet via an outgoing link, all are restarted, and expected to +// forward a response back from the receiver once back online. +// +// The general flow of this test: +// 1. Carol --> Dave --> Alice --> Bob forward payment +// 2. X X X Bob restart sender and intermediaries +// 3. Carol <-- Dave <-- Alice <-- Bob expect settle to propagate +func testSwitchCircuitPersistence(net *lntest.NetworkHarness, t *harnessTest) { + ctxb := context.Background() + + const chanAmt = btcutil.Amount(1000000) + const pushAmt = btcutil.Amount(900000) + var networkChans []*lnrpc.ChannelPoint + + // Open a channel with 100k satoshis between Alice and Bob with Alice + // being the sole funder of the channel. + ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) + chanPointAlice := openChannelAndAssert( + ctxt, t, net, net.Alice, net.Bob, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointAlice) + + aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + aliceFundPoint := wire.OutPoint{ + Hash: *aliceChanTXID, + Index: chanPointAlice.OutputIndex, + } + + // As preliminary setup, we'll create two new nodes: Carol and Dave, + // such that we now have a 4 ndoe, 3 channel topology. Dave will make + // a channel with Alice, and Carol with Dave. After this setup, the + // network topology should now look like: + // Carol -> Dave -> Alice -> Bob + // + // First, we'll create Dave and establish a channel to Alice. + dave := net.NewNode(t.t, "Dave", nil) + defer shutdownAndAssert(net, t, dave) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, dave, net.Alice) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointDave := openChannelAndAssert( + ctxt, t, net, dave, net.Alice, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointDave) + daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + daveFundPoint := wire.OutPoint{ + Hash: *daveChanTXID, + Index: chanPointDave.OutputIndex, + } + + // Next, we'll create Carol and establish a channel to from her to + // Dave. Carol is started in htlchodl mode so that we can disconnect the + // intermediary hops before starting the settle. + carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) + defer shutdownAndAssert(net, t, carol) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, carol, dave) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointCarol := openChannelAndAssert( + ctxt, t, net, carol, dave, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointCarol) + + carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + carolFundPoint := wire.OutPoint{ + Hash: *carolChanTXID, + Index: chanPointCarol.OutputIndex, + } + + // Wait for all nodes to have seen all channels. + nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} + nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} + for _, chanPoint := range networkChans { + for i, node := range nodes { + txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + point := wire.OutPoint{ + Hash: *txid, + Index: chanPoint.OutputIndex, + } + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("%s(%d): timeout waiting for "+ + "channel(%s) open: %v", nodeNames[i], + node.NodeID, point, err) + } + } + } + + // Create 5 invoices for Carol, which expect a payment from Bob for 1k + // satoshis with a different preimage each time. + const numPayments = 5 + const paymentAmt = 1000 + payReqs, _, _, err := createPayReqs( + carol, paymentAmt, numPayments, + ) + if err != nil { + t.Fatalf("unable to create pay reqs: %v", err) + } + + // We'll wait for all parties to recognize the new channels within the + // network. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) + if err != nil { + t.Fatalf("dave didn't advertise his channel: %v", err) + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) + if err != nil { + t.Fatalf("carol didn't advertise her channel in time: %v", + err) + } + + time.Sleep(time.Millisecond * 50) + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + // Wait until all nodes in the network have 5 outstanding htlcs. + var predErr error + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(nodes, numPayments) + if predErr != nil { + return false + } + return true + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Restart the intermediaries and the sender. + if err := net.RestartNode(dave, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + if err := net.RestartNode(net.Alice, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + if err := net.RestartNode(net.Bob, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + // Ensure all of the intermediate links are reconnected. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, net.Alice, dave) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, net.Bob, net.Alice) + + // Ensure all nodes in the network still have 5 outstanding htlcs. + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(nodes, numPayments) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Now restart carol without hodl mode, to settle back the outstanding + // payments. + carol.SetExtraArgs(nil) + if err := net.RestartNode(carol, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, dave, carol) + + // After the payments settle, there should be no active htlcs on any of + // the nodes in the network. + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(nodes, 0) + return predErr == nil + + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // When asserting the amount of satoshis moved, we'll factor in the + // default base fee, as we didn't modify the fee structure when + // creating the seed nodes in the network. + const baseFee = 1 + + // At this point all the channels within our proto network should be + // shifted by 5k satoshis in the direction of Carol, the sink within the + // payment flow generated above. The order of asserts corresponds to + // increasing of time is needed to embed the HTLC in commitment + // transaction, in channel Bob->Alice->David->Carol, order is Carol, + // David, Alice, Bob. + var amountPaid = int64(5000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) + + // Lastly, we will send one more payment to ensure all channels are + // still functioning properly. + finalInvoice := &lnrpc.Invoice{ + Memo: "testing", + Value: paymentAmt, + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + resp, err := carol.AddInvoice(ctxt, finalInvoice) + if err != nil { + t.Fatalf("unable to add invoice: %v", err) + } + + payReqs = []string{resp.PaymentRequest} + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + amountPaid = int64(6000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) + + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) +} + +// testSwitchOfflineDelivery constructs a set of multihop payments, and tests +// that the returning payments are not lost if a peer on the backwards path is +// offline when the settle/fails are received. We expect the payments to be +// buffered in memory, and transmitted as soon as the disconnect link comes back +// online. +// +// The general flow of this test: +// 1. Carol --> Dave --> Alice --> Bob forward payment +// 2. Carol --- Dave X Alice --- Bob disconnect intermediaries +// 3. Carol --- Dave X Alice <-- Bob settle last hop +// 4. Carol <-- Dave <-- Alice --- Bob reconnect, expect settle to propagate +func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { + ctxb := context.Background() + + const chanAmt = btcutil.Amount(1000000) + const pushAmt = btcutil.Amount(900000) + var networkChans []*lnrpc.ChannelPoint + + // Open a channel with 100k satoshis between Alice and Bob with Alice + // being the sole funder of the channel. + ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) + chanPointAlice := openChannelAndAssert( + ctxt, t, net, net.Alice, net.Bob, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointAlice) + + aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + aliceFundPoint := wire.OutPoint{ + Hash: *aliceChanTXID, + Index: chanPointAlice.OutputIndex, + } + + // As preliminary setup, we'll create two new nodes: Carol and Dave, + // such that we now have a 4 ndoe, 3 channel topology. Dave will make + // a channel with Alice, and Carol with Dave. After this setup, the + // network topology should now look like: + // Carol -> Dave -> Alice -> Bob + // + // First, we'll create Dave and establish a channel to Alice. + dave := net.NewNode(t.t, "Dave", nil) + defer shutdownAndAssert(net, t, dave) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, dave, net.Alice) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointDave := openChannelAndAssert( + ctxt, t, net, dave, net.Alice, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointDave) + daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + daveFundPoint := wire.OutPoint{ + Hash: *daveChanTXID, + Index: chanPointDave.OutputIndex, + } + + // Next, we'll create Carol and establish a channel to from her to + // Dave. Carol is started in htlchodl mode so that we can disconnect the + // intermediary hops before starting the settle. + carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) + defer shutdownAndAssert(net, t, carol) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, carol, dave) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointCarol := openChannelAndAssert( + ctxt, t, net, carol, dave, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointCarol) + + carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + carolFundPoint := wire.OutPoint{ + Hash: *carolChanTXID, + Index: chanPointCarol.OutputIndex, + } + + // Wait for all nodes to have seen all channels. + nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} + nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} + for _, chanPoint := range networkChans { + for i, node := range nodes { + txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + point := wire.OutPoint{ + Hash: *txid, + Index: chanPoint.OutputIndex, + } + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("%s(%d): timeout waiting for "+ + "channel(%s) open: %v", nodeNames[i], + node.NodeID, point, err) + } + } + } + + // Create 5 invoices for Carol, which expect a payment from Bob for 1k + // satoshis with a different preimage each time. + const numPayments = 5 + const paymentAmt = 1000 + payReqs, _, _, err := createPayReqs( + carol, paymentAmt, numPayments, + ) + if err != nil { + t.Fatalf("unable to create pay reqs: %v", err) + } + + // We'll wait for all parties to recognize the new channels within the + // network. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) + if err != nil { + t.Fatalf("dave didn't advertise his channel: %v", err) + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) + if err != nil { + t.Fatalf("carol didn't advertise her channel in time: %v", + err) + } + + // Make sure all nodes are fully synced before we continue. + ctxt, cancel := context.WithTimeout(ctxb, defaultTimeout) + defer cancel() + for _, node := range nodes { + err := node.WaitForBlockchainSync(ctxt) + if err != nil { + t.Fatalf("unable to wait for sync: %v", err) + } + } + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + // Wait for all of the payments to reach Carol. + var predErr error + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(nodes, numPayments) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // First, disconnect Dave and Alice so that their link is broken. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + if err := net.DisconnectNodes(ctxt, dave, net.Alice); err != nil { + t.Fatalf("unable to disconnect alice from dave: %v", err) + } + + // Then, reconnect them to ensure Dave doesn't just fail back the htlc. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, dave, net.Alice) + + // Wait to ensure that the payment remain are not failed back after + // reconnecting. All node should report the number payments initiated + // for the duration of the interval. + err = wait.Invariant(func() bool { + predErr = assertNumActiveHtlcs(nodes, numPayments) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc change: %v", predErr) + } + + // Now, disconnect Dave from Alice again before settling back the + // payment. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + if err := net.DisconnectNodes(ctxt, dave, net.Alice); err != nil { + t.Fatalf("unable to disconnect alice from dave: %v", err) + } + + // Now restart carol without hodl mode, to settle back the outstanding + // payments. + carol.SetExtraArgs(nil) + if err := net.RestartNode(carol, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + // Wait for Carol to report no outstanding htlcs. + carolNode := []*lntest.HarnessNode{carol} + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(carolNode, 0) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Make sure all nodes are fully synced again. + ctxt, cancel = context.WithTimeout(ctxb, defaultTimeout) + defer cancel() + for _, node := range nodes { + err := node.WaitForBlockchainSync(ctxt) + if err != nil { + t.Fatalf("unable to wait for sync: %v", err) + } + } + + // Now that the settles have reached Dave, reconnect him with Alice, + // allowing the settles to return to the sender. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, dave, net.Alice) + + // Wait until all outstanding htlcs in the network have been settled. + err = wait.Predicate(func() bool { + return assertNumActiveHtlcs(nodes, 0) == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // When asserting the amount of satoshis moved, we'll factor in the + // default base fee, as we didn't modify the fee structure when + // creating the seed nodes in the network. + const baseFee = 1 + + // At this point all the channels within our proto network should be + // shifted by 5k satoshis in the direction of Carol, the sink within the + // payment flow generated above. The order of asserts corresponds to + // increasing of time is needed to embed the HTLC in commitment + // transaction, in channel Bob->Alice->David->Carol, order is Carol, + // David, Alice, Bob. + var amountPaid = int64(5000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) + + // Lastly, we will send one more payment to ensure all channels are + // still functioning properly. + finalInvoice := &lnrpc.Invoice{ + Memo: "testing", + Value: paymentAmt, + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + resp, err := carol.AddInvoice(ctxt, finalInvoice) + if err != nil { + t.Fatalf("unable to add invoice: %v", err) + } + + payReqs = []string{resp.PaymentRequest} + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + amountPaid = int64(6000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) + + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) +} + +// testSwitchOfflineDeliveryPersistence constructs a set of multihop payments, +// and tests that the returning payments are not lost if a peer on the backwards +// path is offline when the settle/fails are received AND the peer buffering the +// responses is completely restarts. We expect the payments to be reloaded from +// disk, and transmitted as soon as the intermediaries are reconnected. +// +// The general flow of this test: +// 1. Carol --> Dave --> Alice --> Bob forward payment +// 2. Carol --- Dave X Alice --- Bob disconnect intermediaries +// 3. Carol --- Dave X Alice <-- Bob settle last hop +// 4. Carol --- Dave X X Bob restart Alice +// 5. Carol <-- Dave <-- Alice --- Bob expect settle to propagate +func testSwitchOfflineDeliveryPersistence(net *lntest.NetworkHarness, t *harnessTest) { + ctxb := context.Background() + + const chanAmt = btcutil.Amount(1000000) + const pushAmt = btcutil.Amount(900000) + var networkChans []*lnrpc.ChannelPoint + + // Open a channel with 100k satoshis between Alice and Bob with Alice + // being the sole funder of the channel. + ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) + chanPointAlice := openChannelAndAssert( + ctxt, t, net, net.Alice, net.Bob, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointAlice) + + aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + aliceFundPoint := wire.OutPoint{ + Hash: *aliceChanTXID, + Index: chanPointAlice.OutputIndex, + } + + // As preliminary setup, we'll create two new nodes: Carol and Dave, + // such that we now have a 4 ndoe, 3 channel topology. Dave will make + // a channel with Alice, and Carol with Dave. After this setup, the + // network topology should now look like: + // Carol -> Dave -> Alice -> Bob + // + // First, we'll create Dave and establish a channel to Alice. + dave := net.NewNode(t.t, "Dave", nil) + defer shutdownAndAssert(net, t, dave) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, dave, net.Alice) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointDave := openChannelAndAssert( + ctxt, t, net, dave, net.Alice, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + + networkChans = append(networkChans, chanPointDave) + daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + daveFundPoint := wire.OutPoint{ + Hash: *daveChanTXID, + Index: chanPointDave.OutputIndex, + } + + // Next, we'll create Carol and establish a channel to from her to + // Dave. Carol is started in htlchodl mode so that we can disconnect the + // intermediary hops before starting the settle. + carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) + defer shutdownAndAssert(net, t, carol) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, carol, dave) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointCarol := openChannelAndAssert( + ctxt, t, net, carol, dave, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointCarol) + + carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + carolFundPoint := wire.OutPoint{ + Hash: *carolChanTXID, + Index: chanPointCarol.OutputIndex, + } + + // Wait for all nodes to have seen all channels. + nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} + nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} + for _, chanPoint := range networkChans { + for i, node := range nodes { + txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + point := wire.OutPoint{ + Hash: *txid, + Index: chanPoint.OutputIndex, + } + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("%s(%d): timeout waiting for "+ + "channel(%s) open: %v", nodeNames[i], + node.NodeID, point, err) + } + } + } + + // Create 5 invoices for Carol, which expect a payment from Bob for 1k + // satoshis with a different preimage each time. + const numPayments = 5 + const paymentAmt = 1000 + payReqs, _, _, err := createPayReqs( + carol, paymentAmt, numPayments, + ) + if err != nil { + t.Fatalf("unable to create pay reqs: %v", err) + } + + // We'll wait for all parties to recognize the new channels within the + // network. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) + if err != nil { + t.Fatalf("dave didn't advertise his channel: %v", err) + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) + if err != nil { + t.Fatalf("carol didn't advertise her channel in time: %v", + err) + } + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + var predErr error + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(nodes, numPayments) + return predErr == nil + + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Disconnect the two intermediaries, Alice and Dave, by shutting down + // Alice. + if err := net.StopNode(net.Alice); err != nil { + t.Fatalf("unable to shutdown alice: %v", err) + } + + // Now restart carol without hodl mode, to settle back the outstanding + // payments. + carol.SetExtraArgs(nil) + if err := net.RestartNode(carol, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + // Make Carol and Dave are reconnected before waiting for the htlcs to + // clear. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, dave, carol) + + // Wait for Carol to report no outstanding htlcs, and also for Dav to + // receive all the settles from Carol. + carolNode := []*lntest.HarnessNode{carol} + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(carolNode, 0) + if predErr != nil { + return false + } + + predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Finally, restart dave who received the settles, but was unable to + // deliver them to Alice since they were disconnected. + if err := net.RestartNode(dave, nil); err != nil { + t.Fatalf("unable to restart dave: %v", err) + } + if err = net.RestartNode(net.Alice, nil); err != nil { + t.Fatalf("unable to restart alice: %v", err) + } + + // Force Dave and Alice to reconnect before waiting for the htlcs to + // clear. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, dave, net.Alice) + + // After reconnection succeeds, the settles should be propagated all + // the way back to the sender. All nodes should report no active htlcs. + err = wait.Predicate(func() bool { + return assertNumActiveHtlcs(nodes, 0) == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // When asserting the amount of satoshis moved, we'll factor in the + // default base fee, as we didn't modify the fee structure when + // creating the seed nodes in the network. + const baseFee = 1 + + // At this point all the channels within our proto network should be + // shifted by 5k satoshis in the direction of Carol, the sink within the + // payment flow generated above. The order of asserts corresponds to + // increasing of time is needed to embed the HTLC in commitment + // transaction, in channel Bob->Alice->David->Carol, order is Carol, + // David, Alice, Bob. + var amountPaid = int64(5000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) + + // Lastly, we will send one more payment to ensure all channels are + // still functioning properly. + finalInvoice := &lnrpc.Invoice{ + Memo: "testing", + Value: paymentAmt, + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + resp, err := carol.AddInvoice(ctxt, finalInvoice) + if err != nil { + t.Fatalf("unable to add invoice: %v", err) + } + + payReqs = []string{resp.PaymentRequest} + + // Before completing the final payment request, ensure that the + // connection between Dave and Carol has been healed. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, dave, carol) + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + amountPaid = int64(6000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) + + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) +} + +// testSwitchOfflineDeliveryOutgoingOffline constructs a set of multihop payments, +// and tests that the returning payments are not lost if a peer on the backwards +// path is offline when the settle/fails are received AND the peer buffering the +// responses is completely restarts. We expect the payments to be reloaded from +// disk, and transmitted as soon as the intermediaries are reconnected. +// +// The general flow of this test: +// 1. Carol --> Dave --> Alice --> Bob forward payment +// 2. Carol --- Dave X Alice --- Bob disconnect intermediaries +// 3. Carol --- Dave X Alice <-- Bob settle last hop +// 4. Carol --- Dave X X shutdown Bob, restart Alice +// 5. Carol <-- Dave <-- Alice X expect settle to propagate +func testSwitchOfflineDeliveryOutgoingOffline( + net *lntest.NetworkHarness, t *harnessTest) { + ctxb := context.Background() + + const chanAmt = btcutil.Amount(1000000) + const pushAmt = btcutil.Amount(900000) + var networkChans []*lnrpc.ChannelPoint + + // Open a channel with 100k satoshis between Alice and Bob with Alice + // being the sole funder of the channel. + ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) + chanPointAlice := openChannelAndAssert( + ctxt, t, net, net.Alice, net.Bob, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointAlice) + + aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + aliceFundPoint := wire.OutPoint{ + Hash: *aliceChanTXID, + Index: chanPointAlice.OutputIndex, + } + + // As preliminary setup, we'll create two new nodes: Carol and Dave, + // such that we now have a 4 ndoe, 3 channel topology. Dave will make + // a channel with Alice, and Carol with Dave. After this setup, the + // network topology should now look like: + // Carol -> Dave -> Alice -> Bob + // + // First, we'll create Dave and establish a channel to Alice. + dave := net.NewNode(t.t, "Dave", nil) + defer shutdownAndAssert(net, t, dave) + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, dave, net.Alice) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointDave := openChannelAndAssert( + ctxt, t, net, dave, net.Alice, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointDave) + daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + daveFundPoint := wire.OutPoint{ + Hash: *daveChanTXID, + Index: chanPointDave.OutputIndex, + } + + // Next, we'll create Carol and establish a channel to from her to + // Dave. Carol is started in htlchodl mode so that we can disconnect the + // intermediary hops before starting the settle. + carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.ConnectNodes(ctxt, t.t, carol, dave) + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) + + ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) + chanPointCarol := openChannelAndAssert( + ctxt, t, net, carol, dave, + lntest.OpenChannelParams{ + Amt: chanAmt, + PushAmt: pushAmt, + }, + ) + networkChans = append(networkChans, chanPointCarol) + + carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + carolFundPoint := wire.OutPoint{ + Hash: *carolChanTXID, + Index: chanPointCarol.OutputIndex, + } + + // Wait for all nodes to have seen all channels. + nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} + nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} + for _, chanPoint := range networkChans { + for i, node := range nodes { + txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) + if err != nil { + t.Fatalf("unable to get txid: %v", err) + } + point := wire.OutPoint{ + Hash: *txid, + Index: chanPoint.OutputIndex, + } + + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) + if err != nil { + t.Fatalf("%s(%d): timeout waiting for "+ + "channel(%s) open: %v", nodeNames[i], + node.NodeID, point, err) + } + } + } + + // Create 5 invoices for Carol, which expect a payment from Bob for 1k + // satoshis with a different preimage each time. + const numPayments = 5 + const paymentAmt = 1000 + payReqs, _, _, err := createPayReqs( + carol, paymentAmt, numPayments, + ) + if err != nil { + t.Fatalf("unable to create pay reqs: %v", err) + } + + // We'll wait for all parties to recognize the new channels within the + // network. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) + if err != nil { + t.Fatalf("dave didn't advertise his channel: %v", err) + } + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) + if err != nil { + t.Fatalf("carol didn't advertise her channel in time: %v", + err) + } + + // Using Carol as the source, pay to the 5 invoices from Bob created + // above. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + err = completePaymentRequests( + ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, + ) + if err != nil { + t.Fatalf("unable to send payments: %v", err) + } + + // Wait for all payments to reach Carol. + var predErr error + err = wait.Predicate(func() bool { + return assertNumActiveHtlcs(nodes, numPayments) == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Disconnect the two intermediaries, Alice and Dave, so that when carol + // restarts, the response will be held by Dave. + if err := net.StopNode(net.Alice); err != nil { + t.Fatalf("unable to shutdown alice: %v", err) + } + + // Now restart carol without hodl mode, to settle back the outstanding + // payments. + carol.SetExtraArgs(nil) + if err := net.RestartNode(carol, nil); err != nil { + t.Fatalf("Node restart failed: %v", err) + } + + // Wait for Carol to report no outstanding htlcs. + carolNode := []*lntest.HarnessNode{carol} + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(carolNode, 0) + if predErr != nil { + return false + } + + predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // Now check that the total amount was transferred from Dave to Carol. + // The amount transferred should be exactly equal to the invoice total + // payment amount, 5k satsohis. + const amountPaid = int64(5000) + assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, + carolFundPoint, int64(0), amountPaid) + assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, + carolFundPoint, amountPaid, int64(0)) + + // Shutdown carol and leave her offline for the rest of the test. This + // is critical, as we wish to see if Dave can propragate settles even if + // the outgoing link is never revived. + shutdownAndAssert(net, t, carol) + + // Now restart Dave, ensuring he is both persisting the settles, and is + // able to reforward them to Alice after recovering from a restart. + if err := net.RestartNode(dave, nil); err != nil { + t.Fatalf("unable to restart dave: %v", err) + } + if err = net.RestartNode(net.Alice, nil); err != nil { + t.Fatalf("unable to restart alice: %v", err) + } + + // Ensure that Dave is reconnected to Alice before waiting for the + // htlcs to clear. + ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) + net.EnsureConnected(ctxt, t.t, dave, net.Alice) + + // Since Carol has been shutdown permanently, we will wait until all + // other nodes in the network report no active htlcs. + nodesMinusCarol := []*lntest.HarnessNode{net.Bob, net.Alice, dave} + err = wait.Predicate(func() bool { + predErr = assertNumActiveHtlcs(nodesMinusCarol, 0) + return predErr == nil + }, defaultTimeout) + if err != nil { + t.Fatalf("htlc mismatch: %v", predErr) + } + + // When asserting the amount of satoshis moved, we'll factor in the + // default base fee, as we didn't modify the fee structure when + // creating the seed nodes in the network. + const baseFee = 1 + + // At this point, all channels (minus Carol, who is shutdown) should + // show a shift of 5k satoshis towards Carol. The order of asserts + // corresponds to increasing of time is needed to embed the HTLC in + // commitment transaction, in channel Bob->Alice->David, order is + // David, Alice, Bob. + assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, + daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) + assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, + daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, + aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) + assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, + aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) + + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) + ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) + closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) +} diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 526be338..409a3da8 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -5765,1189 +5765,6 @@ func testNodeSignVerify(net *lntest.NetworkHarness, t *harnessTest) { closeChannelAndAssert(ctxt, t, net, net.Alice, aliceBobCh, false) } -// testSwitchCircuitPersistence creates a multihop network to ensure the sender -// and intermediaries are persisting their open payment circuits. After -// forwarding a packet via an outgoing link, all are restarted, and expected to -// forward a response back from the receiver once back online. -// -// The general flow of this test: -// 1. Carol --> Dave --> Alice --> Bob forward payment -// 2. X X X Bob restart sender and intermediaries -// 3. Carol <-- Dave <-- Alice <-- Bob expect settle to propagate -func testSwitchCircuitPersistence(net *lntest.NetworkHarness, t *harnessTest) { - ctxb := context.Background() - - const chanAmt = btcutil.Amount(1000000) - const pushAmt = btcutil.Amount(900000) - var networkChans []*lnrpc.ChannelPoint - - // Open a channel with 100k satoshis between Alice and Bob with Alice - // being the sole funder of the channel. - ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) - chanPointAlice := openChannelAndAssert( - ctxt, t, net, net.Alice, net.Bob, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointAlice) - - aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - aliceFundPoint := wire.OutPoint{ - Hash: *aliceChanTXID, - Index: chanPointAlice.OutputIndex, - } - - // As preliminary setup, we'll create two new nodes: Carol and Dave, - // such that we now have a 4 ndoe, 3 channel topology. Dave will make - // a channel with Alice, and Carol with Dave. After this setup, the - // network topology should now look like: - // Carol -> Dave -> Alice -> Bob - // - // First, we'll create Dave and establish a channel to Alice. - dave := net.NewNode(t.t, "Dave", nil) - defer shutdownAndAssert(net, t, dave) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, dave, net.Alice) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointDave := openChannelAndAssert( - ctxt, t, net, dave, net.Alice, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointDave) - daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - daveFundPoint := wire.OutPoint{ - Hash: *daveChanTXID, - Index: chanPointDave.OutputIndex, - } - - // Next, we'll create Carol and establish a channel to from her to - // Dave. Carol is started in htlchodl mode so that we can disconnect the - // intermediary hops before starting the settle. - carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) - defer shutdownAndAssert(net, t, carol) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, carol, dave) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointCarol := openChannelAndAssert( - ctxt, t, net, carol, dave, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointCarol) - - carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - carolFundPoint := wire.OutPoint{ - Hash: *carolChanTXID, - Index: chanPointCarol.OutputIndex, - } - - // Wait for all nodes to have seen all channels. - nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} - nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} - for _, chanPoint := range networkChans { - for i, node := range nodes { - txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - point := wire.OutPoint{ - Hash: *txid, - Index: chanPoint.OutputIndex, - } - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) - if err != nil { - t.Fatalf("%s(%d): timeout waiting for "+ - "channel(%s) open: %v", nodeNames[i], - node.NodeID, point, err) - } - } - } - - // Create 5 invoices for Carol, which expect a payment from Bob for 1k - // satoshis with a different preimage each time. - const numPayments = 5 - const paymentAmt = 1000 - payReqs, _, _, err := createPayReqs( - carol, paymentAmt, numPayments, - ) - if err != nil { - t.Fatalf("unable to create pay reqs: %v", err) - } - - // We'll wait for all parties to recognize the new channels within the - // network. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) - if err != nil { - t.Fatalf("dave didn't advertise his channel: %v", err) - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) - if err != nil { - t.Fatalf("carol didn't advertise her channel in time: %v", - err) - } - - time.Sleep(time.Millisecond * 50) - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - // Wait until all nodes in the network have 5 outstanding htlcs. - var predErr error - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(nodes, numPayments) - if predErr != nil { - return false - } - return true - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Restart the intermediaries and the sender. - if err := net.RestartNode(dave, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - if err := net.RestartNode(net.Alice, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - if err := net.RestartNode(net.Bob, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - // Ensure all of the intermediate links are reconnected. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, net.Alice, dave) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, net.Bob, net.Alice) - - // Ensure all nodes in the network still have 5 outstanding htlcs. - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(nodes, numPayments) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Now restart carol without hodl mode, to settle back the outstanding - // payments. - carol.SetExtraArgs(nil) - if err := net.RestartNode(carol, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, dave, carol) - - // After the payments settle, there should be no active htlcs on any of - // the nodes in the network. - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(nodes, 0) - return predErr == nil - - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // When asserting the amount of satoshis moved, we'll factor in the - // default base fee, as we didn't modify the fee structure when - // creating the seed nodes in the network. - const baseFee = 1 - - // At this point all the channels within our proto network should be - // shifted by 5k satoshis in the direction of Carol, the sink within the - // payment flow generated above. The order of asserts corresponds to - // increasing of time is needed to embed the HTLC in commitment - // transaction, in channel Bob->Alice->David->Carol, order is Carol, - // David, Alice, Bob. - var amountPaid = int64(5000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) - - // Lastly, we will send one more payment to ensure all channels are - // still functioning properly. - finalInvoice := &lnrpc.Invoice{ - Memo: "testing", - Value: paymentAmt, - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - resp, err := carol.AddInvoice(ctxt, finalInvoice) - if err != nil { - t.Fatalf("unable to add invoice: %v", err) - } - - payReqs = []string{resp.PaymentRequest} - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - amountPaid = int64(6000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) - - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) -} - -// testSwitchOfflineDelivery constructs a set of multihop payments, and tests -// that the returning payments are not lost if a peer on the backwards path is -// offline when the settle/fails are received. We expect the payments to be -// buffered in memory, and transmitted as soon as the disconnect link comes back -// online. -// -// The general flow of this test: -// 1. Carol --> Dave --> Alice --> Bob forward payment -// 2. Carol --- Dave X Alice --- Bob disconnect intermediaries -// 3. Carol --- Dave X Alice <-- Bob settle last hop -// 4. Carol <-- Dave <-- Alice --- Bob reconnect, expect settle to propagate -func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { - ctxb := context.Background() - - const chanAmt = btcutil.Amount(1000000) - const pushAmt = btcutil.Amount(900000) - var networkChans []*lnrpc.ChannelPoint - - // Open a channel with 100k satoshis between Alice and Bob with Alice - // being the sole funder of the channel. - ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) - chanPointAlice := openChannelAndAssert( - ctxt, t, net, net.Alice, net.Bob, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointAlice) - - aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - aliceFundPoint := wire.OutPoint{ - Hash: *aliceChanTXID, - Index: chanPointAlice.OutputIndex, - } - - // As preliminary setup, we'll create two new nodes: Carol and Dave, - // such that we now have a 4 ndoe, 3 channel topology. Dave will make - // a channel with Alice, and Carol with Dave. After this setup, the - // network topology should now look like: - // Carol -> Dave -> Alice -> Bob - // - // First, we'll create Dave and establish a channel to Alice. - dave := net.NewNode(t.t, "Dave", nil) - defer shutdownAndAssert(net, t, dave) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, dave, net.Alice) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointDave := openChannelAndAssert( - ctxt, t, net, dave, net.Alice, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointDave) - daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - daveFundPoint := wire.OutPoint{ - Hash: *daveChanTXID, - Index: chanPointDave.OutputIndex, - } - - // Next, we'll create Carol and establish a channel to from her to - // Dave. Carol is started in htlchodl mode so that we can disconnect the - // intermediary hops before starting the settle. - carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) - defer shutdownAndAssert(net, t, carol) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, carol, dave) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointCarol := openChannelAndAssert( - ctxt, t, net, carol, dave, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointCarol) - - carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - carolFundPoint := wire.OutPoint{ - Hash: *carolChanTXID, - Index: chanPointCarol.OutputIndex, - } - - // Wait for all nodes to have seen all channels. - nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} - nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} - for _, chanPoint := range networkChans { - for i, node := range nodes { - txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - point := wire.OutPoint{ - Hash: *txid, - Index: chanPoint.OutputIndex, - } - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) - if err != nil { - t.Fatalf("%s(%d): timeout waiting for "+ - "channel(%s) open: %v", nodeNames[i], - node.NodeID, point, err) - } - } - } - - // Create 5 invoices for Carol, which expect a payment from Bob for 1k - // satoshis with a different preimage each time. - const numPayments = 5 - const paymentAmt = 1000 - payReqs, _, _, err := createPayReqs( - carol, paymentAmt, numPayments, - ) - if err != nil { - t.Fatalf("unable to create pay reqs: %v", err) - } - - // We'll wait for all parties to recognize the new channels within the - // network. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) - if err != nil { - t.Fatalf("dave didn't advertise his channel: %v", err) - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) - if err != nil { - t.Fatalf("carol didn't advertise her channel in time: %v", - err) - } - - // Make sure all nodes are fully synced before we continue. - ctxt, cancel := context.WithTimeout(ctxb, defaultTimeout) - defer cancel() - for _, node := range nodes { - err := node.WaitForBlockchainSync(ctxt) - if err != nil { - t.Fatalf("unable to wait for sync: %v", err) - } - } - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - // Wait for all of the payments to reach Carol. - var predErr error - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(nodes, numPayments) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // First, disconnect Dave and Alice so that their link is broken. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - if err := net.DisconnectNodes(ctxt, dave, net.Alice); err != nil { - t.Fatalf("unable to disconnect alice from dave: %v", err) - } - - // Then, reconnect them to ensure Dave doesn't just fail back the htlc. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, dave, net.Alice) - - // Wait to ensure that the payment remain are not failed back after - // reconnecting. All node should report the number payments initiated - // for the duration of the interval. - err = wait.Invariant(func() bool { - predErr = assertNumActiveHtlcs(nodes, numPayments) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc change: %v", predErr) - } - - // Now, disconnect Dave from Alice again before settling back the - // payment. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - if err := net.DisconnectNodes(ctxt, dave, net.Alice); err != nil { - t.Fatalf("unable to disconnect alice from dave: %v", err) - } - - // Now restart carol without hodl mode, to settle back the outstanding - // payments. - carol.SetExtraArgs(nil) - if err := net.RestartNode(carol, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - // Wait for Carol to report no outstanding htlcs. - carolNode := []*lntest.HarnessNode{carol} - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(carolNode, 0) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Make sure all nodes are fully synced again. - ctxt, cancel = context.WithTimeout(ctxb, defaultTimeout) - defer cancel() - for _, node := range nodes { - err := node.WaitForBlockchainSync(ctxt) - if err != nil { - t.Fatalf("unable to wait for sync: %v", err) - } - } - - // Now that the settles have reached Dave, reconnect him with Alice, - // allowing the settles to return to the sender. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, dave, net.Alice) - - // Wait until all outstanding htlcs in the network have been settled. - err = wait.Predicate(func() bool { - return assertNumActiveHtlcs(nodes, 0) == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // When asserting the amount of satoshis moved, we'll factor in the - // default base fee, as we didn't modify the fee structure when - // creating the seed nodes in the network. - const baseFee = 1 - - // At this point all the channels within our proto network should be - // shifted by 5k satoshis in the direction of Carol, the sink within the - // payment flow generated above. The order of asserts corresponds to - // increasing of time is needed to embed the HTLC in commitment - // transaction, in channel Bob->Alice->David->Carol, order is Carol, - // David, Alice, Bob. - var amountPaid = int64(5000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) - - // Lastly, we will send one more payment to ensure all channels are - // still functioning properly. - finalInvoice := &lnrpc.Invoice{ - Memo: "testing", - Value: paymentAmt, - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - resp, err := carol.AddInvoice(ctxt, finalInvoice) - if err != nil { - t.Fatalf("unable to add invoice: %v", err) - } - - payReqs = []string{resp.PaymentRequest} - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - amountPaid = int64(6000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) - - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) -} - -// testSwitchOfflineDeliveryPersistence constructs a set of multihop payments, -// and tests that the returning payments are not lost if a peer on the backwards -// path is offline when the settle/fails are received AND the peer buffering the -// responses is completely restarts. We expect the payments to be reloaded from -// disk, and transmitted as soon as the intermediaries are reconnected. -// -// The general flow of this test: -// 1. Carol --> Dave --> Alice --> Bob forward payment -// 2. Carol --- Dave X Alice --- Bob disconnect intermediaries -// 3. Carol --- Dave X Alice <-- Bob settle last hop -// 4. Carol --- Dave X X Bob restart Alice -// 5. Carol <-- Dave <-- Alice --- Bob expect settle to propagate -func testSwitchOfflineDeliveryPersistence(net *lntest.NetworkHarness, t *harnessTest) { - ctxb := context.Background() - - const chanAmt = btcutil.Amount(1000000) - const pushAmt = btcutil.Amount(900000) - var networkChans []*lnrpc.ChannelPoint - - // Open a channel with 100k satoshis between Alice and Bob with Alice - // being the sole funder of the channel. - ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) - chanPointAlice := openChannelAndAssert( - ctxt, t, net, net.Alice, net.Bob, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointAlice) - - aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - aliceFundPoint := wire.OutPoint{ - Hash: *aliceChanTXID, - Index: chanPointAlice.OutputIndex, - } - - // As preliminary setup, we'll create two new nodes: Carol and Dave, - // such that we now have a 4 ndoe, 3 channel topology. Dave will make - // a channel with Alice, and Carol with Dave. After this setup, the - // network topology should now look like: - // Carol -> Dave -> Alice -> Bob - // - // First, we'll create Dave and establish a channel to Alice. - dave := net.NewNode(t.t, "Dave", nil) - defer shutdownAndAssert(net, t, dave) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, dave, net.Alice) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointDave := openChannelAndAssert( - ctxt, t, net, dave, net.Alice, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - - networkChans = append(networkChans, chanPointDave) - daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - daveFundPoint := wire.OutPoint{ - Hash: *daveChanTXID, - Index: chanPointDave.OutputIndex, - } - - // Next, we'll create Carol and establish a channel to from her to - // Dave. Carol is started in htlchodl mode so that we can disconnect the - // intermediary hops before starting the settle. - carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) - defer shutdownAndAssert(net, t, carol) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, carol, dave) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointCarol := openChannelAndAssert( - ctxt, t, net, carol, dave, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointCarol) - - carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - carolFundPoint := wire.OutPoint{ - Hash: *carolChanTXID, - Index: chanPointCarol.OutputIndex, - } - - // Wait for all nodes to have seen all channels. - nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} - nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} - for _, chanPoint := range networkChans { - for i, node := range nodes { - txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - point := wire.OutPoint{ - Hash: *txid, - Index: chanPoint.OutputIndex, - } - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) - if err != nil { - t.Fatalf("%s(%d): timeout waiting for "+ - "channel(%s) open: %v", nodeNames[i], - node.NodeID, point, err) - } - } - } - - // Create 5 invoices for Carol, which expect a payment from Bob for 1k - // satoshis with a different preimage each time. - const numPayments = 5 - const paymentAmt = 1000 - payReqs, _, _, err := createPayReqs( - carol, paymentAmt, numPayments, - ) - if err != nil { - t.Fatalf("unable to create pay reqs: %v", err) - } - - // We'll wait for all parties to recognize the new channels within the - // network. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) - if err != nil { - t.Fatalf("dave didn't advertise his channel: %v", err) - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) - if err != nil { - t.Fatalf("carol didn't advertise her channel in time: %v", - err) - } - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - var predErr error - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(nodes, numPayments) - return predErr == nil - - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Disconnect the two intermediaries, Alice and Dave, by shutting down - // Alice. - if err := net.StopNode(net.Alice); err != nil { - t.Fatalf("unable to shutdown alice: %v", err) - } - - // Now restart carol without hodl mode, to settle back the outstanding - // payments. - carol.SetExtraArgs(nil) - if err := net.RestartNode(carol, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - // Make Carol and Dave are reconnected before waiting for the htlcs to - // clear. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, dave, carol) - - // Wait for Carol to report no outstanding htlcs, and also for Dav to - // receive all the settles from Carol. - carolNode := []*lntest.HarnessNode{carol} - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(carolNode, 0) - if predErr != nil { - return false - } - - predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Finally, restart dave who received the settles, but was unable to - // deliver them to Alice since they were disconnected. - if err := net.RestartNode(dave, nil); err != nil { - t.Fatalf("unable to restart dave: %v", err) - } - if err = net.RestartNode(net.Alice, nil); err != nil { - t.Fatalf("unable to restart alice: %v", err) - } - - // Force Dave and Alice to reconnect before waiting for the htlcs to - // clear. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, dave, net.Alice) - - // After reconnection succeeds, the settles should be propagated all - // the way back to the sender. All nodes should report no active htlcs. - err = wait.Predicate(func() bool { - return assertNumActiveHtlcs(nodes, 0) == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // When asserting the amount of satoshis moved, we'll factor in the - // default base fee, as we didn't modify the fee structure when - // creating the seed nodes in the network. - const baseFee = 1 - - // At this point all the channels within our proto network should be - // shifted by 5k satoshis in the direction of Carol, the sink within the - // payment flow generated above. The order of asserts corresponds to - // increasing of time is needed to embed the HTLC in commitment - // transaction, in channel Bob->Alice->David->Carol, order is Carol, - // David, Alice, Bob. - var amountPaid = int64(5000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) - - // Lastly, we will send one more payment to ensure all channels are - // still functioning properly. - finalInvoice := &lnrpc.Invoice{ - Memo: "testing", - Value: paymentAmt, - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - resp, err := carol.AddInvoice(ctxt, finalInvoice) - if err != nil { - t.Fatalf("unable to add invoice: %v", err) - } - - payReqs = []string{resp.PaymentRequest} - - // Before completing the final payment request, ensure that the - // connection between Dave and Carol has been healed. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, dave, carol) - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - amountPaid = int64(6000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) - - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) -} - -// testSwitchOfflineDeliveryOutgoingOffline constructs a set of multihop payments, -// and tests that the returning payments are not lost if a peer on the backwards -// path is offline when the settle/fails are received AND the peer buffering the -// responses is completely restarts. We expect the payments to be reloaded from -// disk, and transmitted as soon as the intermediaries are reconnected. -// -// The general flow of this test: -// 1. Carol --> Dave --> Alice --> Bob forward payment -// 2. Carol --- Dave X Alice --- Bob disconnect intermediaries -// 3. Carol --- Dave X Alice <-- Bob settle last hop -// 4. Carol --- Dave X X shutdown Bob, restart Alice -// 5. Carol <-- Dave <-- Alice X expect settle to propagate -func testSwitchOfflineDeliveryOutgoingOffline( - net *lntest.NetworkHarness, t *harnessTest) { - ctxb := context.Background() - - const chanAmt = btcutil.Amount(1000000) - const pushAmt = btcutil.Amount(900000) - var networkChans []*lnrpc.ChannelPoint - - // Open a channel with 100k satoshis between Alice and Bob with Alice - // being the sole funder of the channel. - ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) - chanPointAlice := openChannelAndAssert( - ctxt, t, net, net.Alice, net.Bob, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointAlice) - - aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - aliceFundPoint := wire.OutPoint{ - Hash: *aliceChanTXID, - Index: chanPointAlice.OutputIndex, - } - - // As preliminary setup, we'll create two new nodes: Carol and Dave, - // such that we now have a 4 ndoe, 3 channel topology. Dave will make - // a channel with Alice, and Carol with Dave. After this setup, the - // network topology should now look like: - // Carol -> Dave -> Alice -> Bob - // - // First, we'll create Dave and establish a channel to Alice. - dave := net.NewNode(t.t, "Dave", nil) - defer shutdownAndAssert(net, t, dave) - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, dave, net.Alice) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointDave := openChannelAndAssert( - ctxt, t, net, dave, net.Alice, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointDave) - daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - daveFundPoint := wire.OutPoint{ - Hash: *daveChanTXID, - Index: chanPointDave.OutputIndex, - } - - // Next, we'll create Carol and establish a channel to from her to - // Dave. Carol is started in htlchodl mode so that we can disconnect the - // intermediary hops before starting the settle. - carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.ConnectNodes(ctxt, t.t, carol, dave) - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) - - ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) - chanPointCarol := openChannelAndAssert( - ctxt, t, net, carol, dave, - lntest.OpenChannelParams{ - Amt: chanAmt, - PushAmt: pushAmt, - }, - ) - networkChans = append(networkChans, chanPointCarol) - - carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - carolFundPoint := wire.OutPoint{ - Hash: *carolChanTXID, - Index: chanPointCarol.OutputIndex, - } - - // Wait for all nodes to have seen all channels. - nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} - nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} - for _, chanPoint := range networkChans { - for i, node := range nodes { - txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) - if err != nil { - t.Fatalf("unable to get txid: %v", err) - } - point := wire.OutPoint{ - Hash: *txid, - Index: chanPoint.OutputIndex, - } - - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) - if err != nil { - t.Fatalf("%s(%d): timeout waiting for "+ - "channel(%s) open: %v", nodeNames[i], - node.NodeID, point, err) - } - } - } - - // Create 5 invoices for Carol, which expect a payment from Bob for 1k - // satoshis with a different preimage each time. - const numPayments = 5 - const paymentAmt = 1000 - payReqs, _, _, err := createPayReqs( - carol, paymentAmt, numPayments, - ) - if err != nil { - t.Fatalf("unable to create pay reqs: %v", err) - } - - // We'll wait for all parties to recognize the new channels within the - // network. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) - if err != nil { - t.Fatalf("dave didn't advertise his channel: %v", err) - } - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) - if err != nil { - t.Fatalf("carol didn't advertise her channel in time: %v", - err) - } - - // Using Carol as the source, pay to the 5 invoices from Bob created - // above. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - err = completePaymentRequests( - ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, - ) - if err != nil { - t.Fatalf("unable to send payments: %v", err) - } - - // Wait for all payments to reach Carol. - var predErr error - err = wait.Predicate(func() bool { - return assertNumActiveHtlcs(nodes, numPayments) == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Disconnect the two intermediaries, Alice and Dave, so that when carol - // restarts, the response will be held by Dave. - if err := net.StopNode(net.Alice); err != nil { - t.Fatalf("unable to shutdown alice: %v", err) - } - - // Now restart carol without hodl mode, to settle back the outstanding - // payments. - carol.SetExtraArgs(nil) - if err := net.RestartNode(carol, nil); err != nil { - t.Fatalf("Node restart failed: %v", err) - } - - // Wait for Carol to report no outstanding htlcs. - carolNode := []*lntest.HarnessNode{carol} - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(carolNode, 0) - if predErr != nil { - return false - } - - predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // Now check that the total amount was transferred from Dave to Carol. - // The amount transferred should be exactly equal to the invoice total - // payment amount, 5k satsohis. - const amountPaid = int64(5000) - assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, - carolFundPoint, int64(0), amountPaid) - assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, - carolFundPoint, amountPaid, int64(0)) - - // Shutdown carol and leave her offline for the rest of the test. This - // is critical, as we wish to see if Dave can propragate settles even if - // the outgoing link is never revived. - shutdownAndAssert(net, t, carol) - - // Now restart Dave, ensuring he is both persisting the settles, and is - // able to reforward them to Alice after recovering from a restart. - if err := net.RestartNode(dave, nil); err != nil { - t.Fatalf("unable to restart dave: %v", err) - } - if err = net.RestartNode(net.Alice, nil); err != nil { - t.Fatalf("unable to restart alice: %v", err) - } - - // Ensure that Dave is reconnected to Alice before waiting for the - // htlcs to clear. - ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) - net.EnsureConnected(ctxt, t.t, dave, net.Alice) - - // Since Carol has been shutdown permanently, we will wait until all - // other nodes in the network report no active htlcs. - nodesMinusCarol := []*lntest.HarnessNode{net.Bob, net.Alice, dave} - err = wait.Predicate(func() bool { - predErr = assertNumActiveHtlcs(nodesMinusCarol, 0) - return predErr == nil - }, defaultTimeout) - if err != nil { - t.Fatalf("htlc mismatch: %v", predErr) - } - - // When asserting the amount of satoshis moved, we'll factor in the - // default base fee, as we didn't modify the fee structure when - // creating the seed nodes in the network. - const baseFee = 1 - - // At this point, all channels (minus Carol, who is shutdown) should - // show a shift of 5k satoshis towards Carol. The order of asserts - // corresponds to increasing of time is needed to embed the HTLC in - // commitment transaction, in channel Bob->Alice->David, order is - // David, Alice, Bob. - assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, - daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) - assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, - daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, - aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) - assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, - aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) - - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) - ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) - closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) -} - // testSendUpdateDisableChannel ensures that a channel update with the disable // flag set is sent once a channel has been either unilaterally or cooperatively // closed.