package itest import ( "context" "time" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lntest" "github.com/lightningnetwork/lnd/lntest/wait" ) // testSwitchCircuitPersistence creates a multihop network to ensure the sender // and intermediaries are persisting their open payment circuits. After // forwarding a packet via an outgoing link, all are restarted, and expected to // forward a response back from the receiver once back online. // // The general flow of this test: // 1. Carol --> Dave --> Alice --> Bob forward payment // 2. X X X Bob restart sender and intermediaries // 3. Carol <-- Dave <-- Alice <-- Bob expect settle to propagate func testSwitchCircuitPersistence(net *lntest.NetworkHarness, t *harnessTest) { ctxb := context.Background() const chanAmt = btcutil.Amount(1000000) const pushAmt = btcutil.Amount(900000) var networkChans []*lnrpc.ChannelPoint // Open a channel with 100k satoshis between Alice and Bob with Alice // being the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) chanPointAlice := openChannelAndAssert( ctxt, t, net, net.Alice, net.Bob, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointAlice) aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) if err != nil { t.Fatalf("unable to get txid: %v", err) } aliceFundPoint := wire.OutPoint{ Hash: *aliceChanTXID, Index: chanPointAlice.OutputIndex, } // As preliminary setup, we'll create two new nodes: Carol and Dave, // such that we now have a 4 ndoe, 3 channel topology. Dave will make // a channel with Alice, and Carol with Dave. After this setup, the // network topology should now look like: // Carol -> Dave -> Alice -> Bob // // First, we'll create Dave and establish a channel to Alice. dave := net.NewNode(t.t, "Dave", nil) defer shutdownAndAssert(net, t, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, dave, net.Alice) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointDave := openChannelAndAssert( ctxt, t, net, dave, net.Alice, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointDave) daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) if err != nil { t.Fatalf("unable to get txid: %v", err) } daveFundPoint := wire.OutPoint{ Hash: *daveChanTXID, Index: chanPointDave.OutputIndex, } // Next, we'll create Carol and establish a channel to from her to // Dave. Carol is started in htlchodl mode so that we can disconnect the // intermediary hops before starting the settle. carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) defer shutdownAndAssert(net, t, carol) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, carol, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointCarol := openChannelAndAssert( ctxt, t, net, carol, dave, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointCarol) carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) if err != nil { t.Fatalf("unable to get txid: %v", err) } carolFundPoint := wire.OutPoint{ Hash: *carolChanTXID, Index: chanPointCarol.OutputIndex, } // Wait for all nodes to have seen all channels. nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} for _, chanPoint := range networkChans { for i, node := range nodes { txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) if err != nil { t.Fatalf("unable to get txid: %v", err) } point := wire.OutPoint{ Hash: *txid, Index: chanPoint.OutputIndex, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) if err != nil { t.Fatalf("%s(%d): timeout waiting for "+ "channel(%s) open: %v", nodeNames[i], node.NodeID, point, err) } } } // Create 5 invoices for Carol, which expect a payment from Bob for 1k // satoshis with a different preimage each time. const numPayments = 5 const paymentAmt = 1000 payReqs, _, _, err := createPayReqs( carol, paymentAmt, numPayments, ) if err != nil { t.Fatalf("unable to create pay reqs: %v", err) } // We'll wait for all parties to recognize the new channels within the // network. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) if err != nil { t.Fatalf("dave didn't advertise his channel: %v", err) } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) if err != nil { t.Fatalf("carol didn't advertise her channel in time: %v", err) } time.Sleep(time.Millisecond * 50) // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } // Wait until all nodes in the network have 5 outstanding htlcs. var predErr error err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Restart the intermediaries and the sender. if err := net.RestartNode(dave, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } if err := net.RestartNode(net.Alice, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } if err := net.RestartNode(net.Bob, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } // Ensure all of the intermediate links are reconnected. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, net.Alice, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, net.Bob, net.Alice) // Ensure all nodes in the network still have 5 outstanding htlcs. err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Now restart carol without hodl mode, to settle back the outstanding // payments. carol.SetExtraArgs(nil) if err := net.RestartNode(carol, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, dave, carol) // After the payments settle, there should be no active htlcs on any of // the nodes in the network. err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, 0) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // When asserting the amount of satoshis moved, we'll factor in the // default base fee, as we didn't modify the fee structure when // creating the seed nodes in the network. const baseFee = 1 // At this point all the channels within our proto network should be // shifted by 5k satoshis in the direction of Carol, the sink within the // payment flow generated above. The order of asserts corresponds to // increasing of time is needed to embed the HTLC in commitment // transaction, in channel Bob->Alice->David->Carol, order is Carol, // David, Alice, Bob. var amountPaid = int64(5000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) // Lastly, we will send one more payment to ensure all channels are // still functioning properly. finalInvoice := &lnrpc.Invoice{ Memo: "testing", Value: paymentAmt, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) resp, err := carol.AddInvoice(ctxt, finalInvoice) if err != nil { t.Fatalf("unable to add invoice: %v", err) } payReqs = []string{resp.PaymentRequest} // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } amountPaid = int64(6000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) } // testSwitchOfflineDelivery constructs a set of multihop payments, and tests // that the returning payments are not lost if a peer on the backwards path is // offline when the settle/fails are received. We expect the payments to be // buffered in memory, and transmitted as soon as the disconnect link comes back // online. // // The general flow of this test: // 1. Carol --> Dave --> Alice --> Bob forward payment // 2. Carol --- Dave X Alice --- Bob disconnect intermediaries // 3. Carol --- Dave X Alice <-- Bob settle last hop // 4. Carol <-- Dave <-- Alice --- Bob reconnect, expect settle to propagate func testSwitchOfflineDelivery(net *lntest.NetworkHarness, t *harnessTest) { ctxb := context.Background() const chanAmt = btcutil.Amount(1000000) const pushAmt = btcutil.Amount(900000) var networkChans []*lnrpc.ChannelPoint // Open a channel with 100k satoshis between Alice and Bob with Alice // being the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) chanPointAlice := openChannelAndAssert( ctxt, t, net, net.Alice, net.Bob, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointAlice) aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) if err != nil { t.Fatalf("unable to get txid: %v", err) } aliceFundPoint := wire.OutPoint{ Hash: *aliceChanTXID, Index: chanPointAlice.OutputIndex, } // As preliminary setup, we'll create two new nodes: Carol and Dave, // such that we now have a 4 ndoe, 3 channel topology. Dave will make // a channel with Alice, and Carol with Dave. After this setup, the // network topology should now look like: // Carol -> Dave -> Alice -> Bob // // First, we'll create Dave and establish a channel to Alice. dave := net.NewNode(t.t, "Dave", nil) defer shutdownAndAssert(net, t, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, dave, net.Alice) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointDave := openChannelAndAssert( ctxt, t, net, dave, net.Alice, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointDave) daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) if err != nil { t.Fatalf("unable to get txid: %v", err) } daveFundPoint := wire.OutPoint{ Hash: *daveChanTXID, Index: chanPointDave.OutputIndex, } // Next, we'll create Carol and establish a channel to from her to // Dave. Carol is started in htlchodl mode so that we can disconnect the // intermediary hops before starting the settle. carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) defer shutdownAndAssert(net, t, carol) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, carol, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointCarol := openChannelAndAssert( ctxt, t, net, carol, dave, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointCarol) carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) if err != nil { t.Fatalf("unable to get txid: %v", err) } carolFundPoint := wire.OutPoint{ Hash: *carolChanTXID, Index: chanPointCarol.OutputIndex, } // Wait for all nodes to have seen all channels. nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} for _, chanPoint := range networkChans { for i, node := range nodes { txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) if err != nil { t.Fatalf("unable to get txid: %v", err) } point := wire.OutPoint{ Hash: *txid, Index: chanPoint.OutputIndex, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) if err != nil { t.Fatalf("%s(%d): timeout waiting for "+ "channel(%s) open: %v", nodeNames[i], node.NodeID, point, err) } } } // Create 5 invoices for Carol, which expect a payment from Bob for 1k // satoshis with a different preimage each time. const numPayments = 5 const paymentAmt = 1000 payReqs, _, _, err := createPayReqs( carol, paymentAmt, numPayments, ) if err != nil { t.Fatalf("unable to create pay reqs: %v", err) } // We'll wait for all parties to recognize the new channels within the // network. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) if err != nil { t.Fatalf("dave didn't advertise his channel: %v", err) } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) if err != nil { t.Fatalf("carol didn't advertise her channel in time: %v", err) } // Make sure all nodes are fully synced before we continue. ctxt, cancel := context.WithTimeout(ctxb, defaultTimeout) defer cancel() for _, node := range nodes { err := node.WaitForBlockchainSync(ctxt) if err != nil { t.Fatalf("unable to wait for sync: %v", err) } } // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } // Wait for all of the payments to reach Carol. var predErr error err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // First, disconnect Dave and Alice so that their link is broken. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) if err := net.DisconnectNodes(ctxt, dave, net.Alice); err != nil { t.Fatalf("unable to disconnect alice from dave: %v", err) } // Then, reconnect them to ensure Dave doesn't just fail back the htlc. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, dave, net.Alice) // Wait to ensure that the payment remain are not failed back after // reconnecting. All node should report the number payments initiated // for the duration of the interval. err = wait.Invariant(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc change: %v", predErr) } // Now, disconnect Dave from Alice again before settling back the // payment. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) if err := net.DisconnectNodes(ctxt, dave, net.Alice); err != nil { t.Fatalf("unable to disconnect alice from dave: %v", err) } // Now restart carol without hodl mode, to settle back the outstanding // payments. carol.SetExtraArgs(nil) if err := net.RestartNode(carol, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } // Wait for Carol to report no outstanding htlcs. carolNode := []*lntest.HarnessNode{carol} err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(carolNode, 0) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Make sure all nodes are fully synced again. ctxt, cancel = context.WithTimeout(ctxb, defaultTimeout) defer cancel() for _, node := range nodes { err := node.WaitForBlockchainSync(ctxt) if err != nil { t.Fatalf("unable to wait for sync: %v", err) } } // Now that the settles have reached Dave, reconnect him with Alice, // allowing the settles to return to the sender. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, dave, net.Alice) // Wait until all outstanding htlcs in the network have been settled. err = wait.Predicate(func() bool { return assertNumActiveHtlcs(nodes, 0) == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // When asserting the amount of satoshis moved, we'll factor in the // default base fee, as we didn't modify the fee structure when // creating the seed nodes in the network. const baseFee = 1 // At this point all the channels within our proto network should be // shifted by 5k satoshis in the direction of Carol, the sink within the // payment flow generated above. The order of asserts corresponds to // increasing of time is needed to embed the HTLC in commitment // transaction, in channel Bob->Alice->David->Carol, order is Carol, // David, Alice, Bob. var amountPaid = int64(5000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) // Lastly, we will send one more payment to ensure all channels are // still functioning properly. finalInvoice := &lnrpc.Invoice{ Memo: "testing", Value: paymentAmt, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) resp, err := carol.AddInvoice(ctxt, finalInvoice) if err != nil { t.Fatalf("unable to add invoice: %v", err) } payReqs = []string{resp.PaymentRequest} // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } amountPaid = int64(6000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) } // testSwitchOfflineDeliveryPersistence constructs a set of multihop payments, // and tests that the returning payments are not lost if a peer on the backwards // path is offline when the settle/fails are received AND the peer buffering the // responses is completely restarts. We expect the payments to be reloaded from // disk, and transmitted as soon as the intermediaries are reconnected. // // The general flow of this test: // 1. Carol --> Dave --> Alice --> Bob forward payment // 2. Carol --- Dave X Alice --- Bob disconnect intermediaries // 3. Carol --- Dave X Alice <-- Bob settle last hop // 4. Carol --- Dave X X Bob restart Alice // 5. Carol <-- Dave <-- Alice --- Bob expect settle to propagate func testSwitchOfflineDeliveryPersistence(net *lntest.NetworkHarness, t *harnessTest) { ctxb := context.Background() const chanAmt = btcutil.Amount(1000000) const pushAmt = btcutil.Amount(900000) var networkChans []*lnrpc.ChannelPoint // Open a channel with 100k satoshis between Alice and Bob with Alice // being the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) chanPointAlice := openChannelAndAssert( ctxt, t, net, net.Alice, net.Bob, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointAlice) aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) if err != nil { t.Fatalf("unable to get txid: %v", err) } aliceFundPoint := wire.OutPoint{ Hash: *aliceChanTXID, Index: chanPointAlice.OutputIndex, } // As preliminary setup, we'll create two new nodes: Carol and Dave, // such that we now have a 4 ndoe, 3 channel topology. Dave will make // a channel with Alice, and Carol with Dave. After this setup, the // network topology should now look like: // Carol -> Dave -> Alice -> Bob // // First, we'll create Dave and establish a channel to Alice. dave := net.NewNode(t.t, "Dave", nil) defer shutdownAndAssert(net, t, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, dave, net.Alice) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointDave := openChannelAndAssert( ctxt, t, net, dave, net.Alice, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointDave) daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) if err != nil { t.Fatalf("unable to get txid: %v", err) } daveFundPoint := wire.OutPoint{ Hash: *daveChanTXID, Index: chanPointDave.OutputIndex, } // Next, we'll create Carol and establish a channel to from her to // Dave. Carol is started in htlchodl mode so that we can disconnect the // intermediary hops before starting the settle. carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) defer shutdownAndAssert(net, t, carol) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, carol, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointCarol := openChannelAndAssert( ctxt, t, net, carol, dave, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointCarol) carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) if err != nil { t.Fatalf("unable to get txid: %v", err) } carolFundPoint := wire.OutPoint{ Hash: *carolChanTXID, Index: chanPointCarol.OutputIndex, } // Wait for all nodes to have seen all channels. nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} for _, chanPoint := range networkChans { for i, node := range nodes { txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) if err != nil { t.Fatalf("unable to get txid: %v", err) } point := wire.OutPoint{ Hash: *txid, Index: chanPoint.OutputIndex, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) if err != nil { t.Fatalf("%s(%d): timeout waiting for "+ "channel(%s) open: %v", nodeNames[i], node.NodeID, point, err) } } } // Create 5 invoices for Carol, which expect a payment from Bob for 1k // satoshis with a different preimage each time. const numPayments = 5 const paymentAmt = 1000 payReqs, _, _, err := createPayReqs( carol, paymentAmt, numPayments, ) if err != nil { t.Fatalf("unable to create pay reqs: %v", err) } // We'll wait for all parties to recognize the new channels within the // network. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) if err != nil { t.Fatalf("dave didn't advertise his channel: %v", err) } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) if err != nil { t.Fatalf("carol didn't advertise her channel in time: %v", err) } // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } var predErr error err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodes, numPayments) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Disconnect the two intermediaries, Alice and Dave, by shutting down // Alice. if err := net.StopNode(net.Alice); err != nil { t.Fatalf("unable to shutdown alice: %v", err) } // Now restart carol without hodl mode, to settle back the outstanding // payments. carol.SetExtraArgs(nil) if err := net.RestartNode(carol, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } // Make Carol and Dave are reconnected before waiting for the htlcs to // clear. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, dave, carol) // Wait for Carol to report no outstanding htlcs, and also for Dav to // receive all the settles from Carol. carolNode := []*lntest.HarnessNode{carol} err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(carolNode, 0) if predErr != nil { return false } predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Finally, restart dave who received the settles, but was unable to // deliver them to Alice since they were disconnected. if err := net.RestartNode(dave, nil); err != nil { t.Fatalf("unable to restart dave: %v", err) } if err = net.RestartNode(net.Alice, nil); err != nil { t.Fatalf("unable to restart alice: %v", err) } // Force Dave and Alice to reconnect before waiting for the htlcs to // clear. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, dave, net.Alice) // After reconnection succeeds, the settles should be propagated all // the way back to the sender. All nodes should report no active htlcs. err = wait.Predicate(func() bool { return assertNumActiveHtlcs(nodes, 0) == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // When asserting the amount of satoshis moved, we'll factor in the // default base fee, as we didn't modify the fee structure when // creating the seed nodes in the network. const baseFee = 1 // At this point all the channels within our proto network should be // shifted by 5k satoshis in the direction of Carol, the sink within the // payment flow generated above. The order of asserts corresponds to // increasing of time is needed to embed the HTLC in commitment // transaction, in channel Bob->Alice->David->Carol, order is Carol, // David, Alice, Bob. var amountPaid = int64(5000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) // Lastly, we will send one more payment to ensure all channels are // still functioning properly. finalInvoice := &lnrpc.Invoice{ Memo: "testing", Value: paymentAmt, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) resp, err := carol.AddInvoice(ctxt, finalInvoice) if err != nil { t.Fatalf("unable to add invoice: %v", err) } payReqs = []string{resp.PaymentRequest} // Before completing the final payment request, ensure that the // connection between Dave and Carol has been healed. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, dave, carol) // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, true, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } amountPaid = int64(6000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*(numPayments+1))) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*(numPayments+1)), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*(numPayments+1))*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*(numPayments+1))*2, int64(0)) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, carol, chanPointCarol, false) } // testSwitchOfflineDeliveryOutgoingOffline constructs a set of multihop payments, // and tests that the returning payments are not lost if a peer on the backwards // path is offline when the settle/fails are received AND the peer buffering the // responses is completely restarts. We expect the payments to be reloaded from // disk, and transmitted as soon as the intermediaries are reconnected. // // The general flow of this test: // 1. Carol --> Dave --> Alice --> Bob forward payment // 2. Carol --- Dave X Alice --- Bob disconnect intermediaries // 3. Carol --- Dave X Alice <-- Bob settle last hop // 4. Carol --- Dave X X shutdown Bob, restart Alice // 5. Carol <-- Dave <-- Alice X expect settle to propagate func testSwitchOfflineDeliveryOutgoingOffline( net *lntest.NetworkHarness, t *harnessTest) { ctxb := context.Background() const chanAmt = btcutil.Amount(1000000) const pushAmt = btcutil.Amount(900000) var networkChans []*lnrpc.ChannelPoint // Open a channel with 100k satoshis between Alice and Bob with Alice // being the sole funder of the channel. ctxt, _ := context.WithTimeout(ctxb, channelOpenTimeout) chanPointAlice := openChannelAndAssert( ctxt, t, net, net.Alice, net.Bob, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointAlice) aliceChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointAlice) if err != nil { t.Fatalf("unable to get txid: %v", err) } aliceFundPoint := wire.OutPoint{ Hash: *aliceChanTXID, Index: chanPointAlice.OutputIndex, } // As preliminary setup, we'll create two new nodes: Carol and Dave, // such that we now have a 4 ndoe, 3 channel topology. Dave will make // a channel with Alice, and Carol with Dave. After this setup, the // network topology should now look like: // Carol -> Dave -> Alice -> Bob // // First, we'll create Dave and establish a channel to Alice. dave := net.NewNode(t.t, "Dave", nil) defer shutdownAndAssert(net, t, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, dave, net.Alice) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, dave) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointDave := openChannelAndAssert( ctxt, t, net, dave, net.Alice, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointDave) daveChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointDave) if err != nil { t.Fatalf("unable to get txid: %v", err) } daveFundPoint := wire.OutPoint{ Hash: *daveChanTXID, Index: chanPointDave.OutputIndex, } // Next, we'll create Carol and establish a channel to from her to // Dave. Carol is started in htlchodl mode so that we can disconnect the // intermediary hops before starting the settle. carol := net.NewNode(t.t, "Carol", []string{"--hodl.exit-settle"}) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.ConnectNodes(ctxt, t.t, carol, dave) ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.SendCoins(ctxt, t.t, btcutil.SatoshiPerBitcoin, carol) ctxt, _ = context.WithTimeout(ctxb, channelOpenTimeout) chanPointCarol := openChannelAndAssert( ctxt, t, net, carol, dave, lntest.OpenChannelParams{ Amt: chanAmt, PushAmt: pushAmt, }, ) networkChans = append(networkChans, chanPointCarol) carolChanTXID, err := lnrpc.GetChanPointFundingTxid(chanPointCarol) if err != nil { t.Fatalf("unable to get txid: %v", err) } carolFundPoint := wire.OutPoint{ Hash: *carolChanTXID, Index: chanPointCarol.OutputIndex, } // Wait for all nodes to have seen all channels. nodes := []*lntest.HarnessNode{net.Alice, net.Bob, carol, dave} nodeNames := []string{"Alice", "Bob", "Carol", "Dave"} for _, chanPoint := range networkChans { for i, node := range nodes { txid, err := lnrpc.GetChanPointFundingTxid(chanPoint) if err != nil { t.Fatalf("unable to get txid: %v", err) } point := wire.OutPoint{ Hash: *txid, Index: chanPoint.OutputIndex, } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = node.WaitForNetworkChannelOpen(ctxt, chanPoint) if err != nil { t.Fatalf("%s(%d): timeout waiting for "+ "channel(%s) open: %v", nodeNames[i], node.NodeID, point, err) } } } // Create 5 invoices for Carol, which expect a payment from Bob for 1k // satoshis with a different preimage each time. const numPayments = 5 const paymentAmt = 1000 payReqs, _, _, err := createPayReqs( carol, paymentAmt, numPayments, ) if err != nil { t.Fatalf("unable to create pay reqs: %v", err) } // We'll wait for all parties to recognize the new channels within the // network. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = dave.WaitForNetworkChannelOpen(ctxt, chanPointDave) if err != nil { t.Fatalf("dave didn't advertise his channel: %v", err) } ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = carol.WaitForNetworkChannelOpen(ctxt, chanPointCarol) if err != nil { t.Fatalf("carol didn't advertise her channel in time: %v", err) } // Using Carol as the source, pay to the 5 invoices from Bob created // above. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) err = completePaymentRequests( ctxt, net.Bob, net.Bob.RouterClient, payReqs, false, ) if err != nil { t.Fatalf("unable to send payments: %v", err) } // Wait for all payments to reach Carol. var predErr error err = wait.Predicate(func() bool { return assertNumActiveHtlcs(nodes, numPayments) == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Disconnect the two intermediaries, Alice and Dave, so that when carol // restarts, the response will be held by Dave. if err := net.StopNode(net.Alice); err != nil { t.Fatalf("unable to shutdown alice: %v", err) } // Now restart carol without hodl mode, to settle back the outstanding // payments. carol.SetExtraArgs(nil) if err := net.RestartNode(carol, nil); err != nil { t.Fatalf("Node restart failed: %v", err) } // Wait for Carol to report no outstanding htlcs. carolNode := []*lntest.HarnessNode{carol} err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(carolNode, 0) if predErr != nil { return false } predErr = assertNumActiveHtlcsChanPoint(dave, carolFundPoint, 0) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // Now check that the total amount was transferred from Dave to Carol. // The amount transferred should be exactly equal to the invoice total // payment amount, 5k satsohis. const amountPaid = int64(5000) assertAmountPaid(t, "Dave(local) => Carol(remote)", carol, carolFundPoint, int64(0), amountPaid) assertAmountPaid(t, "Dave(local) => Carol(remote)", dave, carolFundPoint, amountPaid, int64(0)) // Shutdown carol and leave her offline for the rest of the test. This // is critical, as we wish to see if Dave can propragate settles even if // the outgoing link is never revived. shutdownAndAssert(net, t, carol) // Now restart Dave, ensuring he is both persisting the settles, and is // able to reforward them to Alice after recovering from a restart. if err := net.RestartNode(dave, nil); err != nil { t.Fatalf("unable to restart dave: %v", err) } if err = net.RestartNode(net.Alice, nil); err != nil { t.Fatalf("unable to restart alice: %v", err) } // Ensure that Dave is reconnected to Alice before waiting for the // htlcs to clear. ctxt, _ = context.WithTimeout(ctxb, defaultTimeout) net.EnsureConnected(ctxt, t.t, dave, net.Alice) // Since Carol has been shutdown permanently, we will wait until all // other nodes in the network report no active htlcs. nodesMinusCarol := []*lntest.HarnessNode{net.Bob, net.Alice, dave} err = wait.Predicate(func() bool { predErr = assertNumActiveHtlcs(nodesMinusCarol, 0) return predErr == nil }, defaultTimeout) if err != nil { t.Fatalf("htlc mismatch: %v", predErr) } // When asserting the amount of satoshis moved, we'll factor in the // default base fee, as we didn't modify the fee structure when // creating the seed nodes in the network. const baseFee = 1 // At this point, all channels (minus Carol, who is shutdown) should // show a shift of 5k satoshis towards Carol. The order of asserts // corresponds to increasing of time is needed to embed the HTLC in // commitment transaction, in channel Bob->Alice->David, order is // David, Alice, Bob. assertAmountPaid(t, "Alice(local) => Dave(remote)", dave, daveFundPoint, int64(0), amountPaid+(baseFee*numPayments)) assertAmountPaid(t, "Alice(local) => Dave(remote)", net.Alice, daveFundPoint, amountPaid+(baseFee*numPayments), int64(0)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Alice, aliceFundPoint, int64(0), amountPaid+((baseFee*numPayments)*2)) assertAmountPaid(t, "Bob(local) => Alice(remote)", net.Bob, aliceFundPoint, amountPaid+(baseFee*numPayments)*2, int64(0)) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, net.Alice, chanPointAlice, false) ctxt, _ = context.WithTimeout(ctxb, channelCloseTimeout) closeChannelAndAssert(ctxt, t, net, dave, chanPointDave, false) }