Merge pull request #3781 from carlaKC/3420-htlcForwarding

htlcswitch: add htlcNotifier
This commit is contained in:
Carla Kirk-Cohen 2020-02-20 14:34:21 +02:00 committed by GitHub
commit 80af2957ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1066 additions and 60 deletions

429
htlcswitch/htlcnotifier.go Normal file

@ -0,0 +1,429 @@
package htlcswitch
import (
"fmt"
"strings"
"sync"
"time"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/htlcswitch/hop"
"github.com/lightningnetwork/lnd/lnwire"
"github.com/lightningnetwork/lnd/subscribe"
)
// HtlcNotifier notifies clients of htlc forwards, failures and settles for
// htlcs that the switch handles. It takes subscriptions for its events and
// notifies them when htlc events occur. These are served on a best-effort
// basis; events are not persisted, delivery is not guaranteed (in the event
// of a crash in the switch, forward events may be lost) and some events may
// be replayed upon restart. Events consumed from this package should be
// de-duplicated by the htlc's unique combination of incoming+outgoing circuit
// and not relied upon for critical operations.
//
// The htlc notifier sends the following kinds of events:
// Forwarding Event:
// - Represents a htlc which is forwarded onward from our node.
// - Present for htlc forwards through our node and local sends.
//
// Link Failure Event:
// - Indicates that a htlc has failed on our incoming or outgoing link,
// with an incoming boolean which indicates where the failure occurred.
// - Incoming link failures are present for failed attempts to pay one of
// our invoices (insufficient amount or mpp timeout, for example) and for
// forwards that we cannot decode to forward onwards.
// - Outgoing link failures are present for forwards or local payments that
// do not meet our outgoing link's policy (insufficient fees, for example)
// and when we fail to forward the payment on (insufficient outgoing
// capacity, or an unknown outgoing link).
//
// Forwarding Failure Event:
// - Forwarding failures indicate that a htlc we forwarded has failed at
// another node down the route.
// - Present for local sends and htlc forwards which fail after they left
// our node.
//
// Settle event:
// - Settle events are present when a htlc which we added is settled through
// the release of a preimage.
// - Present for local receives, and successful local sends or forwards.
//
// Each htlc is identified by its incoming and outgoing circuit key. Htlcs,
// and their subsequent settles or fails, can be identified by the combination
// of incoming and outgoing circuits. Note that receives to our node will
// have a zero outgoing circuit key because the htlc terminates at our
// node, and sends from our node will have a zero incoming circuit key because
// the send originates at our node.
type HtlcNotifier struct {
started sync.Once
stopped sync.Once
// now returns the current time, it is set in the htlcnotifier to allow
// for timestamp mocking in tests.
now func() time.Time
ntfnServer *subscribe.Server
}
// NewHtlcNotifier creates a new HtlcNotifier which gets htlc forwarded,
// failed and settled events from links our node has established with peers
// and sends notifications to subscribing clients.
func NewHtlcNotifier(now func() time.Time) *HtlcNotifier {
return &HtlcNotifier{
now: now,
ntfnServer: subscribe.NewServer(),
}
}
// Start starts the HtlcNotifier and all goroutines it needs to consume
// events and provide subscriptions to clients.
func (h *HtlcNotifier) Start() error {
var err error
h.started.Do(func() {
log.Trace("HtlcNotifier starting")
err = h.ntfnServer.Start()
})
return err
}
// Stop signals the notifier for a graceful shutdown.
func (h *HtlcNotifier) Stop() {
h.stopped.Do(func() {
if err := h.ntfnServer.Stop(); err != nil {
log.Warnf("error stopping htlc notifier: %v", err)
}
})
}
// SubscribeHtlcEvents returns a subscribe.Client that will receive updates
// any time the server is made aware of a new event.
func (h *HtlcNotifier) SubscribeHtlcEvents() (*subscribe.Client, error) {
return h.ntfnServer.Subscribe()
}
// HtlcKey uniquely identifies the htlc.
type HtlcKey struct {
// IncomingCircuit is the channel an htlc id of the incoming htlc.
IncomingCircuit channeldb.CircuitKey
// OutgoingCircuit is the channel and htlc id of the outgoing htlc.
OutgoingCircuit channeldb.CircuitKey
}
// String returns a string representation of a htlc key.
func (k HtlcKey) String() string {
switch {
case k.IncomingCircuit.ChanID == hop.Source:
return k.OutgoingCircuit.String()
case k.OutgoingCircuit.ChanID == hop.Exit:
return k.IncomingCircuit.String()
default:
return fmt.Sprintf("%v -> %v", k.IncomingCircuit,
k.OutgoingCircuit)
}
}
// HtlcInfo provides the details of a htlc that our node has processed. For
// forwards, incoming and outgoing values are set, whereas sends and receives
// will only have outgoing or incoming details set.
type HtlcInfo struct {
// IncomingTimelock is the time lock of the htlc on our incoming
// channel.
IncomingTimeLock uint32
// OutgoingTimelock is the time lock the htlc on our outgoing channel.
OutgoingTimeLock uint32
// IncomingAmt is the amount of the htlc on our incoming channel.
IncomingAmt lnwire.MilliSatoshi
// OutgoingAmt is the amount of the htlc on our outgoing channel.
OutgoingAmt lnwire.MilliSatoshi
}
// String returns a string representation of a htlc.
func (h HtlcInfo) String() string {
var details []string
// If the incoming information is not zero, as is the case for a send,
// we include the incoming amount and timelock.
if h.IncomingAmt != 0 || h.IncomingTimeLock != 0 {
str := fmt.Sprintf("incoming amount: %v, "+
"incoming timelock: %v", h.IncomingAmt,
h.IncomingTimeLock)
details = append(details, str)
}
// If the outgoing information is not zero, as is the case for a
// receive, we include the outgoing amount and timelock.
if h.OutgoingAmt != 0 || h.OutgoingTimeLock != 0 {
str := fmt.Sprintf("outgoing amount: %v, "+
"outgoing timelock: %v", h.OutgoingAmt,
h.OutgoingTimeLock)
details = append(details, str)
}
return strings.Join(details, ", ")
}
// HtlcEventType represents the type of event that a htlc was part of.
type HtlcEventType int
const (
// HtlcEventTypeSend represents a htlc that was part of a send from
// our node.
HtlcEventTypeSend HtlcEventType = iota
// HtlcEventTypeReceive represents a htlc that was part of a receive
// to our node.
HtlcEventTypeReceive
// HtlcEventTypeForward represents a htlc that was forwarded through
// our node.
HtlcEventTypeForward
)
// String returns a string representation of a htlc event type.
func (h HtlcEventType) String() string {
switch h {
case HtlcEventTypeSend:
return "send"
case HtlcEventTypeReceive:
return "receive"
case HtlcEventTypeForward:
return "forward"
default:
return "unknown"
}
}
// ForwardingEvent represents a htlc that was forwarded onwards from our node.
// Sends which originate from our node will report forward events with zero
// incoming circuits in their htlc key.
type ForwardingEvent struct {
// HtlcKey uniquely identifies the htlc, and can be used to match the
// forwarding event with subsequent settle/fail events.
HtlcKey
// HtlcInfo contains details about the htlc.
HtlcInfo
// HtlcEventType classifies the event as part of a local send or
// receive, or as part of a forward.
HtlcEventType
// Timestamp is the time when this htlc was forwarded.
Timestamp time.Time
}
// LinkFailEvent describes a htlc that failed on our incoming or outgoing
// link. The incoming bool is true for failures on incoming links, and false
// for failures on outgoing links. The failure reason is provided by a lnwire
// failure message which is enriched with a failure detail in the cases where
// the wire failure message does not contain full information about the
// failure.
type LinkFailEvent struct {
// HtlcKey uniquely identifies the htlc.
HtlcKey
// HtlcInfo contains details about the htlc.
HtlcInfo
// HtlcEventType classifies the event as part of a local send or
// receive, or as part of a forward.
HtlcEventType
// LinkError is the reason that we failed the htlc.
LinkError *LinkError
// Incoming is true if the htlc was failed on an incoming link.
// If it failed on the outgoing link, it is false.
Incoming bool
// Timestamp is the time when the link failure occurred.
Timestamp time.Time
}
// ForwardingFailEvent represents a htlc failure which occurred down the line
// after we forwarded a htlc onwards. An error is not included in this event
// because errors returned down the route are encrypted. HtlcInfo is not
// reliably available for forwarding failures, so it is omitted. These events
// should be matched with their corresponding forward event to obtain this
// information.
type ForwardingFailEvent struct {
// HtlcKey uniquely identifies the htlc, and can be used to match the
// htlc with its corresponding forwarding event.
HtlcKey
// HtlcEventType classifies the event as part of a local send or
// receive, or as part of a forward.
HtlcEventType
// Timestamp is the time when the forwarding failure was received.
Timestamp time.Time
}
// SettleEvent represents a htlc that was settled. HtlcInfo is not reliably
// available for forwarding failures, so it is omitted. These events should
// be matched with corresponding forward events or invoices (for receives)
// to obtain additional information about the htlc.
type SettleEvent struct {
// HtlcKey uniquely identifies the htlc, and can be used to match
// forwards with their corresponding forwarding event.
HtlcKey
// HtlcEventType classifies the event as part of a local send or
// receive, or as part of a forward.
HtlcEventType
// Timestamp is the time when this htlc was settled.
Timestamp time.Time
}
// NotifyForwardingEvent notifies the HtlcNotifier than a htlc has been
// forwarded.
//
// Note this is part of the htlcNotifier interface.
func (h *HtlcNotifier) NotifyForwardingEvent(key HtlcKey, info HtlcInfo,
eventType HtlcEventType) {
event := &ForwardingEvent{
HtlcKey: key,
HtlcInfo: info,
HtlcEventType: eventType,
Timestamp: h.now(),
}
log.Tracef("Notifying forward event: %v over %v, %v", eventType, key,
info)
if err := h.ntfnServer.SendUpdate(event); err != nil {
log.Warnf("Unable to send forwarding event: %v", err)
}
}
// NotifyLinkFailEvent notifies that a htlc has failed on our incoming
// or outgoing link.
//
// Note this is part of the htlcNotifier interface.
func (h *HtlcNotifier) NotifyLinkFailEvent(key HtlcKey, info HtlcInfo,
eventType HtlcEventType, linkErr *LinkError, incoming bool) {
event := &LinkFailEvent{
HtlcKey: key,
HtlcInfo: info,
HtlcEventType: eventType,
LinkError: linkErr,
Incoming: incoming,
Timestamp: h.now(),
}
log.Tracef("Notifying link failure event: %v over %v, %v", eventType,
key, info)
if err := h.ntfnServer.SendUpdate(event); err != nil {
log.Warnf("Unable to send link fail event: %v", err)
}
}
// NotifyForwardingFailEvent notifies the HtlcNotifier that a htlc we
// forwarded has failed down the line.
//
// Note this is part of the htlcNotifier interface.
func (h *HtlcNotifier) NotifyForwardingFailEvent(key HtlcKey,
eventType HtlcEventType) {
event := &ForwardingFailEvent{
HtlcKey: key,
HtlcEventType: eventType,
Timestamp: h.now(),
}
log.Tracef("Notifying forwarding failure event: %v over %v", eventType,
key)
if err := h.ntfnServer.SendUpdate(event); err != nil {
log.Warnf("Unable to send forwarding fail event: %v", err)
}
}
// NotifySettleEvent notifies the HtlcNotifier that a htlc that we committed
// to as part of a forward or a receive to our node has been settled.
//
// Note this is part of the htlcNotifier interface.
func (h *HtlcNotifier) NotifySettleEvent(key HtlcKey, eventType HtlcEventType) {
event := &SettleEvent{
HtlcKey: key,
HtlcEventType: eventType,
Timestamp: h.now(),
}
log.Tracef("Notifying settle event: %v over %v", eventType, key)
if err := h.ntfnServer.SendUpdate(event); err != nil {
log.Warnf("Unable to send settle event: %v", err)
}
}
// newHtlc key returns a htlc key for the packet provided. If the packet
// has a zero incoming channel ID, the packet is for one of our own sends,
// which has the payment id stashed in the incoming htlc id. If this is the
// case, we replace the incoming htlc id with zero so that the notifier
// consistently reports zero circuit keys for events that terminate or
// originate at our node.
func newHtlcKey(pkt *htlcPacket) HtlcKey {
htlcKey := HtlcKey{
IncomingCircuit: channeldb.CircuitKey{
ChanID: pkt.incomingChanID,
HtlcID: pkt.incomingHTLCID,
},
OutgoingCircuit: CircuitKey{
ChanID: pkt.outgoingChanID,
HtlcID: pkt.outgoingHTLCID,
},
}
// If the packet has a zero incoming channel ID, it is a send that was
// initiated at our node. If this is the case, our internal pid is in
// the incoming htlc ID, so we overwrite it with 0 for notification
// purposes.
if pkt.incomingChanID == hop.Source {
htlcKey.IncomingCircuit.HtlcID = 0
}
return htlcKey
}
// newHtlcInfo returns HtlcInfo for the packet provided.
func newHtlcInfo(pkt *htlcPacket) HtlcInfo {
return HtlcInfo{
IncomingTimeLock: pkt.incomingTimeout,
OutgoingTimeLock: pkt.outgoingTimeout,
IncomingAmt: pkt.incomingAmount,
OutgoingAmt: pkt.amount,
}
}
// getEventType returns the htlc type based on the fields set in the htlc
// packet. Sends that originate at our node have the source (zero) incoming
// channel ID. Receives to our node have the exit (zero) outgoing channel ID
// and forwards have both fields set.
func getEventType(pkt *htlcPacket) HtlcEventType {
switch {
case pkt.incomingChanID == hop.Source:
return HtlcEventTypeSend
case pkt.outgoingChanID == hop.Exit:
return HtlcEventTypeReceive
default:
return HtlcEventTypeForward
}
}

@ -180,3 +180,29 @@ type TowerClient interface {
// isTweakless should be true.
BackupState(*lnwire.ChannelID, *lnwallet.BreachRetribution, bool) error
}
// htlcNotifier is an interface which represents the input side of the
// HtlcNotifier which htlc events are piped through. This interface is intended
// to allow for mocking of the htlcNotifier in tests, so is unexported because
// it is not needed outside of the htlcSwitch package.
type htlcNotifier interface {
// NotifyForwardingEvent notifies the HtlcNotifier than a htlc has been
// forwarded.
NotifyForwardingEvent(key HtlcKey, info HtlcInfo,
eventType HtlcEventType)
// NotifyIncomingLinkFailEvent notifies that a htlc has failed on our
// incoming link. It takes an isReceive bool to differentiate between
// our node's receives and forwards.
NotifyLinkFailEvent(key HtlcKey, info HtlcInfo,
eventType HtlcEventType, linkErr *LinkError, incoming bool)
// NotifyForwardingFailEvent notifies the HtlcNotifier that a htlc we
// forwarded has failed down the line.
NotifyForwardingFailEvent(key HtlcKey, eventType HtlcEventType)
// NotifySettleEvent notifies the HtlcNotifier that a htlc that we
// committed to as part of a forward or a receive to our node has been
// settled.
NotifySettleEvent(key HtlcKey, eventType HtlcEventType)
}

@ -272,6 +272,10 @@ type ChannelLinkConfig struct {
// NotifyInactiveChannel allows the switch to tell the ChannelNotifier
// when channels become inactive.
NotifyInactiveChannel func(wire.OutPoint)
// HtlcNotifier is an instance of a htlcNotifier which we will pipe htlc
// events through.
HtlcNotifier htlcNotifier
}
// channelLink is the service which drives a channel's commitment update
@ -1204,10 +1208,7 @@ func (l *channelLink) processHtlcResolution(resolution invoices.HtlcResolution,
l.log.Debugf("received settle resolution for %v"+
"with outcome: %v", circuitKey, res.Outcome)
return l.settleHTLC(
res.Preimage, htlc.pd.HtlcIndex,
htlc.pd.SourceRef,
)
return l.settleHTLC(res.Preimage, htlc.pd)
// For htlc failures, we get the relevant failure message based
// on the failure resolution and then fail the htlc.
@ -1220,8 +1221,7 @@ func (l *channelLink) processHtlcResolution(resolution invoices.HtlcResolution,
failure := getResolutionFailure(res, htlc.pd.Amount)
l.sendHTLCError(
htlc.pd.HtlcIndex, failure,
htlc.obfuscator, htlc.pd.SourceRef,
htlc.pd, failure, htlc.obfuscator, true,
)
return nil
@ -1414,6 +1414,18 @@ func (l *channelLink) handleDownStreamPkt(pkt *htlcPacket, isReProcess bool) {
l.cfg.Peer.SendMessage(false, htlc)
// Send a forward event notification to htlcNotifier.
l.cfg.HtlcNotifier.NotifyForwardingEvent(
newHtlcKey(pkt),
HtlcInfo{
IncomingTimeLock: pkt.incomingTimeout,
IncomingAmt: pkt.incomingAmount,
OutgoingTimeLock: htlc.Expiry,
OutgoingAmt: htlc.Amount,
},
getEventType(pkt),
)
case *lnwire.UpdateFulfillHTLC:
// If hodl.SettleOutgoing mode is active, we exit early to
// simulate arbitrary delays between the switch adding the
@ -1472,6 +1484,12 @@ func (l *channelLink) handleDownStreamPkt(pkt *htlcPacket, isReProcess bool) {
l.cfg.Peer.SendMessage(false, htlc)
isSettle = true
// Send a settle event notification to htlcNotifier.
l.cfg.HtlcNotifier.NotifySettleEvent(
newHtlcKey(pkt),
getEventType(pkt),
)
case *lnwire.UpdateFailHTLC:
// If hodl.FailOutgoing mode is active, we exit early to
// simulate arbitrary delays between the switch adding a FAIL to
@ -1525,10 +1543,28 @@ func (l *channelLink) handleDownStreamPkt(pkt *htlcPacket, isReProcess bool) {
htlc.ChanID = l.ChanID()
htlc.ID = pkt.incomingHTLCID
// Finally, we send the HTLC message to the peer which
// initially created the HTLC.
// We send the HTLC message to the peer which initially created
// the HTLC.
l.cfg.Peer.SendMessage(false, htlc)
isSettle = true
// If the packet does not have a link failure set, it failed
// further down the route so we notify a forwarding failure.
// Otherwise, we notify a link failure because it failed at our
// node.
if pkt.linkFailure != nil {
l.cfg.HtlcNotifier.NotifyLinkFailEvent(
newHtlcKey(pkt),
newHtlcInfo(pkt),
getEventType(pkt),
pkt.linkFailure,
false,
)
} else {
l.cfg.HtlcNotifier.NotifyForwardingFailEvent(
newHtlcKey(pkt), getEventType(pkt),
)
}
}
// If this newly added update exceeds the min batch size for adds, or
@ -2656,9 +2692,7 @@ func (l *channelLink) processRemoteAdds(fwdPkg *channeldb.FwdPkg,
// later date
failure := lnwire.NewInvalidOnionPayload(failedType, 0)
l.sendHTLCError(
pd.HtlcIndex,
NewLinkError(failure),
obfuscator, pd.SourceRef,
pd, NewLinkError(failure), obfuscator, false,
)
l.log.Errorf("unable to decode forwarding "+
@ -2771,10 +2805,7 @@ func (l *channelLink) processRemoteAdds(fwdPkg *channeldb.FwdPkg,
)
l.sendHTLCError(
pd.HtlcIndex,
NewLinkError(failure),
obfuscator,
pd.SourceRef,
pd, NewLinkError(failure), obfuscator, false,
)
continue
}
@ -2860,7 +2891,7 @@ func (l *channelLink) processExitHop(pd *lnwallet.PaymentDescriptor,
failure := NewLinkError(
lnwire.NewFinalIncorrectHtlcAmount(pd.Amount),
)
l.sendHTLCError(pd.HtlcIndex, failure, obfuscator, pd.SourceRef)
l.sendHTLCError(pd, failure, obfuscator, true)
return nil
}
@ -2875,7 +2906,7 @@ func (l *channelLink) processExitHop(pd *lnwallet.PaymentDescriptor,
failure := NewLinkError(
lnwire.NewFinalIncorrectCltvExpiry(pd.Timeout),
)
l.sendHTLCError(pd.HtlcIndex, failure, obfuscator, pd.SourceRef)
l.sendHTLCError(pd, failure, obfuscator, true)
return nil
}
@ -2916,15 +2947,15 @@ func (l *channelLink) processExitHop(pd *lnwallet.PaymentDescriptor,
}
// settleHTLC settles the HTLC on the channel.
func (l *channelLink) settleHTLC(preimage lntypes.Preimage, htlcIndex uint64,
sourceRef *channeldb.AddRef) error {
func (l *channelLink) settleHTLC(preimage lntypes.Preimage,
pd *lnwallet.PaymentDescriptor) error {
hash := preimage.Hash()
l.log.Infof("settling htlc %v as exit hop", hash)
err := l.channel.SettleHTLC(
preimage, htlcIndex, sourceRef, nil, nil,
preimage, pd.HtlcIndex, pd.SourceRef, nil, nil,
)
if err != nil {
return fmt.Errorf("unable to settle htlc: %v", err)
@ -2942,10 +2973,21 @@ func (l *channelLink) settleHTLC(preimage lntypes.Preimage, htlcIndex uint64,
// remote peer.
l.cfg.Peer.SendMessage(false, &lnwire.UpdateFulfillHTLC{
ChanID: l.ChanID(),
ID: htlcIndex,
ID: pd.HtlcIndex,
PaymentPreimage: preimage,
})
// Once we have successfully settled the htlc, notify a settle event.
l.cfg.HtlcNotifier.NotifySettleEvent(
HtlcKey{
IncomingCircuit: channeldb.CircuitKey{
ChanID: l.ShortChanID(),
HtlcID: pd.HtlcIndex,
},
},
HtlcEventTypeReceive,
)
return nil
}
@ -2991,8 +3033,8 @@ func (l *channelLink) handleBatchFwdErrs(errChan chan error) {
// sendHTLCError functions cancels HTLC and send cancel message back to the
// peer from which HTLC was received.
func (l *channelLink) sendHTLCError(htlcIndex uint64, failure *LinkError,
e hop.ErrorEncrypter, sourceRef *channeldb.AddRef) {
func (l *channelLink) sendHTLCError(pd *lnwallet.PaymentDescriptor,
failure *LinkError, e hop.ErrorEncrypter, isReceive bool) {
reason, err := e.EncryptFirstHop(failure.WireMessage())
if err != nil {
@ -3000,7 +3042,7 @@ func (l *channelLink) sendHTLCError(htlcIndex uint64, failure *LinkError,
return
}
err = l.channel.FailHTLC(htlcIndex, reason, sourceRef, nil, nil)
err = l.channel.FailHTLC(pd.HtlcIndex, reason, pd.SourceRef, nil, nil)
if err != nil {
l.log.Errorf("unable cancel htlc: %v", err)
return
@ -3008,9 +3050,35 @@ func (l *channelLink) sendHTLCError(htlcIndex uint64, failure *LinkError,
l.cfg.Peer.SendMessage(false, &lnwire.UpdateFailHTLC{
ChanID: l.ChanID(),
ID: htlcIndex,
ID: pd.HtlcIndex,
Reason: reason,
})
// Notify a link failure on our incoming link. Outgoing htlc information
// is not available at this point, because we have not decrypted the
// onion, so it is excluded.
var eventType HtlcEventType
if isReceive {
eventType = HtlcEventTypeReceive
} else {
eventType = HtlcEventTypeForward
}
l.cfg.HtlcNotifier.NotifyLinkFailEvent(
HtlcKey{
IncomingCircuit: channeldb.CircuitKey{
ChanID: l.ShortChanID(),
HtlcID: pd.HtlcIndex,
},
},
HtlcInfo{
IncomingTimeLock: pd.Timeout,
IncomingAmt: pd.Amount,
},
eventType,
failure,
true,
)
}
// sendMalformedHTLCError helper function which sends the malformed HTLC update

@ -1750,6 +1750,7 @@ func newSingleLinkTestHarness(chanAmt, chanReserve btcutil.Amount) (
MaxFeeAllocation: DefaultMaxLinkFeeAllocation,
NotifyActiveChannel: func(wire.OutPoint) {},
NotifyInactiveChannel: func(wire.OutPoint) {},
HtlcNotifier: aliceSwitch.cfg.HtlcNotifier,
}
aliceLink := NewChannelLink(aliceCfg, aliceLc.channel)
@ -4313,6 +4314,7 @@ func (h *persistentLinkHarness) restartLink(
MaxFeeAllocation: DefaultMaxLinkFeeAllocation,
NotifyActiveChannel: func(wire.OutPoint) {},
NotifyInactiveChannel: func(wire.OutPoint) {},
HtlcNotifier: aliceSwitch.cfg.HtlcNotifier,
}
aliceLink := NewChannelLink(aliceCfg, aliceChannel)
@ -5523,6 +5525,7 @@ func TestCheckHtlcForward(t *testing.T) {
},
FetchLastChannelUpdate: fetchLastChannelUpdate,
MaxOutgoingCltvExpiry: DefaultMaxOutgoingCltvExpiry,
HtlcNotifier: &mockHTLCNotifier{},
},
log: log,
channel: testChannel.channel,

@ -176,6 +176,7 @@ func initSwitchWithDB(startingHeight uint32, db *channeldb.DB) (*Switch, error)
FwdEventTicker: ticker.NewForce(DefaultFwdEventInterval),
LogEventTicker: ticker.NewForce(DefaultLogInterval),
AckEventTicker: ticker.NewForce(DefaultAckInterval),
HtlcNotifier: &mockHTLCNotifier{},
}
return New(cfg, startingHeight)
@ -1009,3 +1010,22 @@ func (m *mockOnionErrorDecryptor) DecryptError(encryptedData []byte) (
Message: m.message,
}, m.err
}
var _ htlcNotifier = (*mockHTLCNotifier)(nil)
type mockHTLCNotifier struct{}
func (h *mockHTLCNotifier) NotifyForwardingEvent(key HtlcKey, info HtlcInfo,
eventType HtlcEventType) {
}
func (h *mockHTLCNotifier) NotifyLinkFailEvent(key HtlcKey, info HtlcInfo,
eventType HtlcEventType, linkErr *LinkError, incoming bool) {
}
func (h *mockHTLCNotifier) NotifyForwardingFailEvent(key HtlcKey,
eventType HtlcEventType) {
}
func (h *mockHTLCNotifier) NotifySettleEvent(key HtlcKey, eventType HtlcEventType) {
}

@ -150,6 +150,10 @@ type Config struct {
// the switch when a new block has arrived.
Notifier chainntnfs.ChainNotifier
// HtlcNotifier is an instance of a htlcNotifier which we will pipe htlc
// events through.
HtlcNotifier htlcNotifier
// FwdEventTicker is a signal that instructs the htlcswitch to flush any
// pending forwarding events.
FwdEventTicker ticker.Ticker
@ -764,38 +768,23 @@ func (s *Switch) handleLocalDispatch(pkt *htlcPacket) error {
// User have created the htlc update therefore we should find the
// appropriate channel link and send the payment over this link.
if htlc, ok := pkt.htlc.(*lnwire.UpdateAddHTLC); ok {
// Try to find links by node destination.
s.indexMtx.RLock()
link, err := s.getLinkByShortID(pkt.outgoingChanID)
s.indexMtx.RUnlock()
link, err := s.handleLocalAddHTLC(pkt, htlc)
if err != nil {
log.Errorf("Link %v not found", pkt.outgoingChanID)
return NewLinkError(&lnwire.FailUnknownNextPeer{})
}
if !link.EligibleToForward() {
log.Errorf("Link %v is not available to forward",
pkt.outgoingChanID)
// The update does not need to be populated as the error
// will be returned back to the router.
return NewDetailedLinkError(
lnwire.NewTemporaryChannelFailure(nil),
OutgoingFailureLinkNotEligible,
// Notify the htlc notifier of a link failure on our
// outgoing link. Incoming timelock/amount values are
// not set because they are not present for local sends.
s.cfg.HtlcNotifier.NotifyLinkFailEvent(
newHtlcKey(pkt),
HtlcInfo{
OutgoingTimeLock: htlc.Expiry,
OutgoingAmt: htlc.Amount,
},
HtlcEventTypeSend,
err,
false,
)
}
// Ensure that the htlc satisfies the outgoing channel policy.
currentHeight := atomic.LoadUint32(&s.bestHeight)
htlcErr := link.CheckHtlcTransit(
htlc.PaymentHash,
htlc.Amount,
htlc.Expiry, currentHeight,
)
if htlcErr != nil {
log.Errorf("Link %v policy for local forward not "+
"satisfied", pkt.outgoingChanID)
return htlcErr
return err
}
return link.HandleSwitchPacket(pkt)
@ -807,6 +796,47 @@ func (s *Switch) handleLocalDispatch(pkt *htlcPacket) error {
return nil
}
// handleLocalAddHTLC handles the addition of a htlc for a send that
// originates from our node. It returns the link that the htlc should
// be forwarded outwards on, and a link error if the htlc cannot be
// forwarded.
func (s *Switch) handleLocalAddHTLC(pkt *htlcPacket,
htlc *lnwire.UpdateAddHTLC) (ChannelLink, *LinkError) {
// Try to find links by node destination.
s.indexMtx.RLock()
link, err := s.getLinkByShortID(pkt.outgoingChanID)
s.indexMtx.RUnlock()
if err != nil {
log.Errorf("Link %v not found", pkt.outgoingChanID)
return nil, NewLinkError(&lnwire.FailUnknownNextPeer{})
}
if !link.EligibleToForward() {
log.Errorf("Link %v is not available to forward",
pkt.outgoingChanID)
// The update does not need to be populated as the error
// will be returned back to the router.
return nil, NewDetailedLinkError(
lnwire.NewTemporaryChannelFailure(nil),
OutgoingFailureLinkNotEligible,
)
}
// Ensure that the htlc satisfies the outgoing channel policy.
currentHeight := atomic.LoadUint32(&s.bestHeight)
htlcErr := link.CheckHtlcTransit(
htlc.PaymentHash, htlc.Amount, htlc.Expiry, currentHeight,
)
if htlcErr != nil {
log.Errorf("Link %v policy for local forward not "+
"satisfied", pkt.outgoingChanID)
return nil, htlcErr
}
return link, nil
}
// handleLocalResponse processes a Settle or Fail responding to a
// locally-initiated payment. This is handled asynchronously to avoid blocking
// the main event loop within the switch, as these operations can require
@ -868,6 +898,18 @@ func (s *Switch) handleLocalResponse(pkt *htlcPacket) {
pkt.inKey(), err)
return
}
// Finally, notify on the htlc failure or success that has been handled.
key := newHtlcKey(pkt)
eventType := getEventType(pkt)
switch pkt.htlc.(type) {
case *lnwire.UpdateFulfillHTLC:
s.cfg.HtlcNotifier.NotifySettleEvent(key, eventType)
case *lnwire.UpdateFailHTLC:
s.cfg.HtlcNotifier.NotifyForwardingFailEvent(key, eventType)
}
}
// extractResult uses the given deobfuscator to extract the payment result from
@ -1252,12 +1294,21 @@ func (s *Switch) failAddPacket(packet *htlcPacket, failure *LinkError) error {
log.Error(failure.Error())
// Create a failure packet for this htlc. The the full set of
// information about the htlc failure is included so that they can
// be included in link failure notifications.
failPkt := &htlcPacket{
sourceRef: packet.sourceRef,
incomingChanID: packet.incomingChanID,
incomingHTLCID: packet.incomingHTLCID,
circuit: packet.circuit,
linkFailure: failure,
sourceRef: packet.sourceRef,
incomingChanID: packet.incomingChanID,
incomingHTLCID: packet.incomingHTLCID,
outgoingChanID: packet.outgoingChanID,
outgoingHTLCID: packet.outgoingHTLCID,
incomingAmount: packet.incomingAmount,
amount: packet.amount,
incomingTimeout: packet.incomingTimeout,
outgoingTimeout: packet.outgoingTimeout,
circuit: packet.circuit,
linkFailure: failure,
htlc: &lnwire.UpdateFailHTLC{
Reason: reason,
},

@ -14,11 +14,14 @@ import (
"github.com/btcsuite/fastsha256"
"github.com/davecgh/go-spew/spew"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/htlcswitch/hop"
"github.com/lightningnetwork/lnd/lntypes"
"github.com/lightningnetwork/lnd/lnwire"
"github.com/lightningnetwork/lnd/ticker"
)
var zeroCircuit = channeldb.CircuitKey{}
func genPreimage() ([32]byte, error) {
var preimage [32]byte
if _, err := io.ReadFull(rand.Reader, preimage[:]); err != nil {
@ -2697,3 +2700,361 @@ func TestInvalidFailure(t *testing.T) {
t.Fatal("err wasn't received")
}
}
// htlcNotifierEvents is a function that generates a set of expected htlc
// notifier evetns for each node in a three hop network with the dynamic
// values provided. These functions take dynamic values so that changes to
// external systems (such as our default timelock delta) do not break
// these tests.
type htlcNotifierEvents func(channels *clusterChannels, htlcID uint64,
ts time.Time, htlc *lnwire.UpdateAddHTLC,
hops []*hop.Payload) ([]interface{}, []interface{}, []interface{})
// TestHtlcNotifier tests the notifying of htlc events that are routed over a
// three hop network. It sets up an Alice -> Bob -> Carol network and routes
// payments from Alice -> Carol to test events from the perspective of a
// sending (Alice), forwarding (Bob) and receiving (Carol) node. Test cases
// are present for saduccessful and failed payments.
func TestHtlcNotifier(t *testing.T) {
tests := []struct {
name string
// Options is a set of options to apply to the three hop
// network's servers.
options []serverOption
// expectedEvents is a function which returns an expected set
// of events for the test.
expectedEvents htlcNotifierEvents
// iterations is the number of times we will send a payment,
// this is used to send more than one payment to force non-
// zero htlc indexes to make sure we aren't just checking
// default values.
iterations int
}{
{
name: "successful three hop payment",
options: nil,
expectedEvents: func(channels *clusterChannels,
htlcID uint64, ts time.Time,
htlc *lnwire.UpdateAddHTLC,
hops []*hop.Payload) ([]interface{},
[]interface{}, []interface{}) {
return getThreeHopEvents(
channels, htlcID, ts, htlc, hops, nil,
)
},
iterations: 2,
},
{
name: "failed at forwarding link",
// Set a functional option which disables bob as a
// forwarding node to force a payment error.
options: []serverOption{
serverOptionRejectHtlc(false, true, false),
},
expectedEvents: func(channels *clusterChannels,
htlcID uint64, ts time.Time,
htlc *lnwire.UpdateAddHTLC,
hops []*hop.Payload) ([]interface{},
[]interface{}, []interface{}) {
return getThreeHopEvents(
channels, htlcID, ts, htlc, hops,
&LinkError{
msg: &lnwire.FailChannelDisabled{},
FailureDetail: OutgoingFailureForwardsDisabled,
},
)
},
iterations: 1,
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
testHtcNotifier(
t, test.options, test.iterations,
test.expectedEvents,
)
})
}
}
// testHtcNotifier runs a htlc notifier test.
func testHtcNotifier(t *testing.T, testOpts []serverOption, iterations int,
getEvents htlcNotifierEvents) {
t.Parallel()
// First, we'll create our traditional three hop
// network.
channels, cleanUp, _, err := createClusterChannels(
btcutil.SatoshiPerBitcoin*3,
btcutil.SatoshiPerBitcoin*5)
if err != nil {
t.Fatalf("unable to create channel: %v", err)
}
defer cleanUp()
// Mock time so that all events are reported with a static timestamp.
now := time.Now()
mockTime := func() time.Time {
return now
}
// Create htlc notifiers for each server in the three hop network and
// start them.
aliceNotifier := NewHtlcNotifier(mockTime)
if err := aliceNotifier.Start(); err != nil {
t.Fatalf("could not start alice notifier")
}
defer aliceNotifier.Stop()
bobNotifier := NewHtlcNotifier(mockTime)
if err := bobNotifier.Start(); err != nil {
t.Fatalf("could not start bob notifier")
}
defer bobNotifier.Stop()
carolNotifier := NewHtlcNotifier(mockTime)
if err := carolNotifier.Start(); err != nil {
t.Fatalf("could not start carol notifier")
}
defer carolNotifier.Stop()
// Create a notifier server option which will set our htlc notifiers
// for the three hop network.
notifierOption := serverOptionWithHtlcNotifier(
aliceNotifier, bobNotifier, carolNotifier,
)
// Add the htlcNotifier option to any other options
// set in the test.
options := append(testOpts, notifierOption)
n := newThreeHopNetwork(
t, channels.aliceToBob,
channels.bobToAlice, channels.bobToCarol,
channels.carolToBob, testStartingHeight,
options...,
)
if err := n.start(); err != nil {
t.Fatalf("unable to start three hop "+
"network: %v", err)
}
defer n.stop()
// Before we forward anything, subscribe to htlc events
// from each notifier.
aliceEvents, err := aliceNotifier.SubscribeHtlcEvents()
if err != nil {
t.Fatalf("could not subscribe to alice's"+
" events: %v", err)
}
defer aliceEvents.Cancel()
bobEvents, err := bobNotifier.SubscribeHtlcEvents()
if err != nil {
t.Fatalf("could not subscribe to bob's"+
" events: %v", err)
}
defer bobEvents.Cancel()
carolEvents, err := carolNotifier.SubscribeHtlcEvents()
if err != nil {
t.Fatalf("could not subscribe to carol's"+
" events: %v", err)
}
defer carolEvents.Cancel()
// Send multiple payments, as specified by the test to test incrementing
// of htlc ids.
for i := 0; i < iterations; i++ {
// We'll start off by making a payment from
// Alice -> Bob -> Carol.
htlc, hops := n.sendThreeHopPayment(t)
alice, bob, carol := getEvents(
channels, uint64(i), now, htlc, hops,
)
checkHtlcEvents(t, aliceEvents.Updates(), alice)
checkHtlcEvents(t, bobEvents.Updates(), bob)
checkHtlcEvents(t, carolEvents.Updates(), carol)
}
}
// checkHtlcEvents checks that a subscription has the set of htlc events
// we expect it to have.
func checkHtlcEvents(t *testing.T, events <-chan interface{},
expectedEvents []interface{}) {
for _, expected := range expectedEvents {
select {
case event := <-events:
if !reflect.DeepEqual(event, expected) {
t.Fatalf("expected %v, got: %v", expected,
event)
}
case <-time.After(time.Second):
t.Fatalf("expected event: %v", expected)
}
}
}
// sendThreeHopPayment is a helper function which sends a payment over
// Alice -> Bob -> Carol in a three hop network and returns Alice's first htlc
// and the remainder of the hops.
func (n *threeHopNetwork) sendThreeHopPayment(t *testing.T) (*lnwire.UpdateAddHTLC,
[]*hop.Payload) {
amount := lnwire.NewMSatFromSatoshis(btcutil.SatoshiPerBitcoin)
htlcAmt, totalTimelock, hops := generateHops(amount, testStartingHeight,
n.firstBobChannelLink, n.carolChannelLink)
blob, err := generateRoute(hops...)
if err != nil {
t.Fatal(err)
}
invoice, htlc, pid, err := generatePayment(
amount, htlcAmt, totalTimelock, blob,
)
if err != nil {
t.Fatal(err)
}
err = n.carolServer.registry.AddInvoice(*invoice, htlc.PaymentHash)
if err != nil {
t.Fatalf("unable to add invoice in carol registry: %v", err)
}
if err := n.aliceServer.htlcSwitch.SendHTLC(
n.firstBobChannelLink.ShortChanID(), pid, htlc,
); err != nil {
t.Fatalf("could not send htlc")
}
return htlc, hops
}
// getThreeHopEvents gets the set of htlc events that we expect for a payment
// from Alice -> Bob -> Carol. If a non-nil link error is provided, the set
// of events will fail on Bob's outgoing link.
func getThreeHopEvents(channels *clusterChannels, htlcID uint64,
ts time.Time, htlc *lnwire.UpdateAddHTLC, hops []*hop.Payload,
linkError *LinkError) ([]interface{}, []interface{}, []interface{}) {
aliceKey := HtlcKey{
IncomingCircuit: zeroCircuit,
OutgoingCircuit: channeldb.CircuitKey{
ChanID: channels.aliceToBob.ShortChanID(),
HtlcID: htlcID,
},
}
// Alice always needs a forwarding event because she initiates the
// send.
aliceEvents := []interface{}{
&ForwardingEvent{
HtlcKey: aliceKey,
HtlcInfo: HtlcInfo{
OutgoingTimeLock: htlc.Expiry,
OutgoingAmt: htlc.Amount,
},
HtlcEventType: HtlcEventTypeSend,
Timestamp: ts,
},
}
bobKey := HtlcKey{
IncomingCircuit: channeldb.CircuitKey{
ChanID: channels.bobToAlice.ShortChanID(),
HtlcID: htlcID,
},
OutgoingCircuit: channeldb.CircuitKey{
ChanID: channels.bobToCarol.ShortChanID(),
HtlcID: htlcID,
},
}
bobInfo := HtlcInfo{
IncomingTimeLock: htlc.Expiry,
IncomingAmt: htlc.Amount,
OutgoingTimeLock: hops[1].FwdInfo.OutgoingCTLV,
OutgoingAmt: hops[1].FwdInfo.AmountToForward,
}
// If we expect the payment to fail, we add failures for alice and
// bob, and no events for carol because the payment never reaches her.
if linkError != nil {
aliceEvents = append(aliceEvents,
&ForwardingFailEvent{
HtlcKey: aliceKey,
HtlcEventType: HtlcEventTypeSend,
Timestamp: ts,
},
)
bobEvents := []interface{}{
&LinkFailEvent{
HtlcKey: bobKey,
HtlcInfo: bobInfo,
HtlcEventType: HtlcEventTypeForward,
LinkError: linkError,
Incoming: false,
Timestamp: ts,
},
}
return aliceEvents, bobEvents, nil
}
// If we want to get events for a successful payment, we add a settle
// for alice, a forward and settle for bob and a receive settle for
// carol.
aliceEvents = append(
aliceEvents,
&SettleEvent{
HtlcKey: aliceKey,
HtlcEventType: HtlcEventTypeSend,
Timestamp: ts,
},
)
bobEvents := []interface{}{
&ForwardingEvent{
HtlcKey: bobKey,
HtlcInfo: bobInfo,
HtlcEventType: HtlcEventTypeForward,
Timestamp: ts,
},
&SettleEvent{
HtlcKey: bobKey,
HtlcEventType: HtlcEventTypeForward,
Timestamp: ts,
},
}
carolEvents := []interface{}{
&SettleEvent{
HtlcKey: HtlcKey{
IncomingCircuit: channeldb.CircuitKey{
ChanID: channels.carolToBob.ShortChanID(),
HtlcID: htlcID,
},
OutgoingCircuit: zeroCircuit,
},
HtlcEventType: HtlcEventTypeReceive,
Timestamp: ts,
},
}
return aliceEvents, bobEvents, carolEvents
}

@ -966,9 +966,11 @@ func createClusterChannels(aliceToBob, bobToCarol btcutil.Amount) (
// alice first bob second bob carol
// channel link channel link channel link channel link
//
// This function takes server options which can be used to apply custom
// settings to alice, bob and carol.
func newThreeHopNetwork(t testing.TB, aliceChannel, firstBobChannel,
secondBobChannel, carolChannel *lnwallet.LightningChannel,
startingHeight uint32) *threeHopNetwork {
startingHeight uint32, opts ...serverOption) *threeHopNetwork {
aliceDb := aliceChannel.State().Db
bobDb := firstBobChannel.State().Db
@ -996,6 +998,12 @@ func newThreeHopNetwork(t testing.TB, aliceChannel, firstBobChannel,
t.Fatalf("unable to create carol server: %v", err)
}
// Apply all additional functional options to the servers before
// creating any links.
for _, option := range opts {
option(aliceServer, bobServer, carolServer)
}
// Create mock decoder instead of sphinx one in order to mock the route
// which htlc should follow.
aliceDecoder := newMockIteratorDecoder()
@ -1045,6 +1053,34 @@ func newThreeHopNetwork(t testing.TB, aliceChannel, firstBobChannel,
}
}
// serverOption is a function which alters the three servers created for
// a three hop network to allow custom settings on each server.
type serverOption func(aliceServer, bobServer, carolServer *mockServer)
// serverOptionWithHtlcNotifier is a functional option for the creation of
// three hop network servers which allows setting of htlc notifiers.
// Note that these notifiers should be started and stopped by the calling
// function.
func serverOptionWithHtlcNotifier(alice, bob,
carol *HtlcNotifier) serverOption {
return func(aliceServer, bobServer, carolServer *mockServer) {
aliceServer.htlcSwitch.cfg.HtlcNotifier = alice
bobServer.htlcSwitch.cfg.HtlcNotifier = bob
carolServer.htlcSwitch.cfg.HtlcNotifier = carol
}
}
// serverOptionRejectHtlc is the functional option for setting the reject
// htlc config option in each server's switch.
func serverOptionRejectHtlc(alice, bob, carol bool) serverOption {
return func(aliceServer, bobServer, carolServer *mockServer) {
aliceServer.htlcSwitch.cfg.RejectHTLC = alice
bobServer.htlcSwitch.cfg.RejectHTLC = bob
carolServer.htlcSwitch.cfg.RejectHTLC = carol
}
}
// createTwoClusterChannels creates lightning channels which are needed for
// a 2 hop network cluster to be initialized.
func createTwoClusterChannels(aliceToBob, bobToCarol btcutil.Amount) (
@ -1139,6 +1175,7 @@ func (h *hopNetwork) createChannelLink(server, peer *mockServer,
MaxFeeAllocation: DefaultMaxLinkFeeAllocation,
NotifyActiveChannel: func(wire.OutPoint) {},
NotifyInactiveChannel: func(wire.OutPoint) {},
HtlcNotifier: server.htlcSwitch.cfg.HtlcNotifier,
},
channel,
)

@ -636,6 +636,7 @@ func (p *peer) addLink(chanPoint *wire.OutPoint,
MaxFeeAllocation: cfg.MaxChannelFeeAllocation,
NotifyActiveChannel: p.server.channelNotifier.NotifyActiveChannelEvent,
NotifyInactiveChannel: p.server.channelNotifier.NotifyInactiveChannelEvent,
HtlcNotifier: p.server.htlcNotifier,
}
link := htlcswitch.NewChannelLink(linkCfg, lnChan)

@ -203,6 +203,8 @@ type server struct {
peerNotifier *peernotifier.PeerNotifier
htlcNotifier *htlcswitch.HtlcNotifier
witnessBeacon contractcourt.WitnessBeacon
breachArbiter *breachArbiter
@ -438,6 +440,8 @@ func newServer(listenAddrs []net.Addr, chanDB *channeldb.DB,
return nil, err
}
s.htlcNotifier = htlcswitch.NewHtlcNotifier(time.Now)
s.htlcSwitch, err = htlcswitch.New(htlcswitch.Config{
DB: chanDB,
LocalChannelClose: func(pubKey []byte,
@ -467,6 +471,7 @@ func newServer(listenAddrs []net.Addr, chanDB *channeldb.DB,
ExtractErrorEncrypter: s.sphinx.ExtractErrorEncrypter,
FetchLastChannelUpdate: s.fetchLastChanUpdate(),
Notifier: s.cc.chainNotifier,
HtlcNotifier: s.htlcNotifier,
FwdEventTicker: ticker.New(htlcswitch.DefaultFwdEventInterval),
LogEventTicker: ticker.New(htlcswitch.DefaultLogInterval),
AckEventTicker: ticker.New(htlcswitch.DefaultAckInterval),
@ -1265,6 +1270,10 @@ func (s *server) Start() error {
startErr = err
return
}
if err := s.htlcNotifier.Start(); err != nil {
startErr = err
return
}
if err := s.sphinx.Start(); err != nil {
startErr = err
return
@ -1429,6 +1438,7 @@ func (s *server) Stop() error {
s.sweeper.Stop()
s.channelNotifier.Stop()
s.peerNotifier.Stop()
s.htlcNotifier.Stop()
s.cc.wallet.Shutdown()
s.cc.chainView.Stop()
s.connMgr.Stop()