routing: use probability source in path finding

This PR replaces the previously used edge and node ignore lists in path
finding by a probability based system. It modifies path finding so that
it not only compares routes on fee and time lock, but also takes route
success probability into account.

Allowing routes to be compared based on success probability is achieved
by introducing a 'virtual' cost of a payment attempt and using that to
translate probability into another cost factor.
This commit is contained in:
Joost Jager 2019-03-19 11:45:10 +01:00
parent b6102ad191
commit 6b70791c2d
No known key found for this signature in database
GPG Key ID: A61B9D4C393C59C7
7 changed files with 327 additions and 118 deletions

@ -123,9 +123,21 @@ func (r *RouterBackend) QueryRoutes(ctx context.Context,
}
restrictions := &routing.RestrictParams{
FeeLimit: feeLimit,
IgnoredNodes: ignoredNodes,
IgnoredEdges: ignoredEdges,
FeeLimit: feeLimit,
ProbabilitySource: func(node route.Vertex,
edge routing.EdgeLocator) float64 {
if _, ok := ignoredNodes[node]; ok {
return 0
}
if _, ok := ignoredEdges[edge]; ok {
return 0
}
return 1
},
PaymentAttemptPenalty: routing.DefaultPaymentAttemptPenalty,
}
// Query the channel router for a possible path to the destination that

@ -39,6 +39,11 @@ func TestQueryRoutes(t *testing.T) {
t.Fatal(err)
}
ignoredEdge := routing.EdgeLocator{
ChannelID: 555,
Direction: 1,
}
request := &lnrpc.QueryRoutesRequest{
PubKey: destKey,
Amt: 100000,
@ -75,22 +80,22 @@ func TestQueryRoutes(t *testing.T) {
t.Fatal("unexpected fee limit")
}
if len(restrictions.IgnoredEdges) != 1 {
t.Fatal("unexpected ignored edges map size")
if restrictions.ProbabilitySource(route.Vertex{},
ignoredEdge,
) != 0 {
t.Fatal("expecting 0% probability for ignored edge")
}
if _, ok := restrictions.IgnoredEdges[routing.EdgeLocator{
ChannelID: 555, Direction: 1,
}]; !ok {
t.Fatal("unexpected ignored edge")
if restrictions.ProbabilitySource(ignoreNodeVertex,
routing.EdgeLocator{},
) != 0 {
t.Fatal("expecting 0% probability for ignored node")
}
if len(restrictions.IgnoredNodes) != 1 {
t.Fatal("unexpected ignored nodes map size")
}
if _, ok := restrictions.IgnoredNodes[ignoreNodeVertex]; !ok {
t.Fatal("unexpected ignored node")
if restrictions.ProbabilitySource(route.Vertex{},
routing.EdgeLocator{},
) != 1 {
t.Fatal("expecting 100% probability")
}
hops := []*route.Hop{{}}

@ -25,8 +25,14 @@ type nodeWithDist struct {
// node. This value does not include the final cltv.
incomingCltv uint32
// fee is the fee that this node is charging for forwarding.
fee lnwire.MilliSatoshi
// probability is the probability that from this node onward the route
// is successful.
probability float64
// weight is the cost of the route from this node to the destination.
// Includes the routing fees and a virtual cost factor to account for
// time locks.
weight int64
}
// distanceHeap is a min-distance heap that's used within our path finding

@ -40,6 +40,14 @@ type pathFinder = func(g *graphParams, r *RestrictParams,
source, target route.Vertex, amt lnwire.MilliSatoshi) (
[]*channeldb.ChannelEdgePolicy, error)
var (
// DefaultPaymentAttemptPenalty is the virtual cost in path finding weight
// units of executing a payment attempt that fails. It is used to trade
// off potentially better routes against their probability of
// succeeding.
DefaultPaymentAttemptPenalty = lnwire.NewMSatFromSatoshis(100)
)
// edgePolicyWithSource is a helper struct to keep track of the source node
// of a channel edge. ChannelEdgePolicy only contains to destination node
// of the edge.
@ -228,13 +236,9 @@ type graphParams struct {
// RestrictParams wraps the set of restrictions passed to findPath that the
// found path must adhere to.
type RestrictParams struct {
// IgnoredNodes is an optional set of nodes that should be ignored if
// encountered during path finding.
IgnoredNodes map[route.Vertex]struct{}
// IgnoredEdges is an optional set of edges that should be ignored if
// encountered during path finding.
IgnoredEdges map[EdgeLocator]struct{}
// ProbabilitySource is a callback that is expected to return the
// success probability of traversing the channel from the node.
ProbabilitySource func(route.Vertex, EdgeLocator) float64
// FeeLimit is a maximum fee amount allowed to be used on the path from
// the source to the target.
@ -248,6 +252,12 @@ type RestrictParams struct {
// ctlv. After path finding is complete, the caller needs to increase
// all cltv expiry heights with the required final cltv delta.
CltvLimit *uint32
// PaymentAttemptPenalty is the virtual cost in path finding weight
// units of executing a payment attempt that fails. It is used to trade
// off potentially better routes against their probability of
// succeeding.
PaymentAttemptPenalty lnwire.MilliSatoshi
}
// findPath attempts to find a path from the source node within the
@ -331,10 +341,11 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
targetNode := &channeldb.LightningNode{PubKeyBytes: target}
distance[target] = nodeWithDist{
dist: 0,
weight: 0,
node: targetNode,
amountToReceive: amt,
fee: 0,
incomingCltv: 0,
probability: 1,
}
// We'll use this map as a series of "next" hop pointers. So to get
@ -342,15 +353,6 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
// mapped to within `next`.
next := make(map[route.Vertex]*channeldb.ChannelEdgePolicy)
ignoredEdges := r.IgnoredEdges
if ignoredEdges == nil {
ignoredEdges = make(map[EdgeLocator]struct{})
}
ignoredNodes := r.IgnoredNodes
if ignoredNodes == nil {
ignoredNodes = make(map[route.Vertex]struct{})
}
// processEdge is a helper closure that will be used to make sure edges
// satisfy our specific requirements.
processEdge := func(fromNode *channeldb.LightningNode,
@ -380,21 +382,26 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
return
}
// If this vertex or edge has been black listed, then we'll
// skip exploring this edge.
if _, ok := ignoredNodes[fromVertex]; ok {
return
}
locator := newEdgeLocator(edge)
if _, ok := ignoredEdges[*locator]; ok {
return
}
// Calculate amount that the candidate node would have to sent
// out.
toNodeDist := distance[toNode]
amountToSend := toNodeDist.amountToReceive
// Request the success probability for this edge.
locator := newEdgeLocator(edge)
edgeProbability := r.ProbabilitySource(
fromVertex, *locator,
)
log.Tracef("path finding probability: fromnode=%v, chanid=%v, "+
"probability=%v", fromVertex, locator.ChannelID,
edgeProbability)
// If the probability is zero, there is no point in trying.
if edgeProbability == 0 {
return
}
// If the estimated bandwidth of the channel edge is not able
// to carry the amount that needs to be send, return.
if bandwidth < amountToSend {
@ -453,19 +460,32 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
return
}
// Calculate total probability of successfully reaching target
// by multiplying the probabilities. Both this edge and the rest
// of the route must succeed.
probability := toNodeDist.probability * edgeProbability
// By adding fromNode in the route, there will be an extra
// weight composed of the fee that this node will charge and
// the amount that will be locked for timeLockDelta blocks in
// the HTLC that is handed out to fromNode.
weight := edgeWeight(amountToReceive, fee, timeLockDelta)
// Compute the tentative distance to this new channel/edge
// which is the distance from our toNode to the target node
// Compute the tentative weight to this new channel/edge
// which is the weight from our toNode to the target node
// plus the weight of this edge.
tempDist := toNodeDist.dist + weight
tempWeight := toNodeDist.weight + weight
// If this new tentative distance is not better than the current
// best known distance to this node, return.
// Add an extra factor to the weight to take into account the
// probability.
tempDist := getProbabilityBasedDist(
tempWeight, probability, int64(r.PaymentAttemptPenalty),
)
// If the current best route is better than this candidate
// route, return. It is important to also return if the distance
// is equal, because otherwise the algorithm could run into an
// endless loop.
if tempDist >= distance[fromVertex].dist {
return
}
@ -483,10 +503,11 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
// map is populated with this edge.
distance[fromVertex] = nodeWithDist{
dist: tempDist,
weight: tempWeight,
node: fromNode,
amountToReceive: amountToReceive,
fee: fee,
incomingCltv: incomingCltv,
probability: probability,
}
next[fromVertex] = edge
@ -614,5 +635,53 @@ func findPath(g *graphParams, r *RestrictParams, source, target route.Vertex,
"too many hops")
}
log.Debugf("Found route: probability=%v, hops=%v, fee=%v\n",
distance[source].probability, numEdges,
distance[source].amountToReceive-amt)
return pathEdges, nil
}
// getProbabilityBasedDist converts a weight into a distance that takes into
// account the success probability and the (virtual) cost of a failed payment
// attempt.
//
// Derivation:
//
// Suppose there are two routes A and B with fees Fa and Fb and success
// probabilities Pa and Pb.
//
// Is the expected cost of trying route A first and then B lower than trying the
// other way around?
//
// The expected cost of A-then-B is: Pa*Fa + (1-Pa)*Pb*(c+Fb)
//
// The expected cost of B-then-A is: Pb*Fb + (1-Pb)*Pa*(c+Fa)
//
// In these equations, the term representing the case where both A and B fail is
// left out because its value would be the same in both cases.
//
// Pa*Fa + (1-Pa)*Pb*(c+Fb) < Pb*Fb + (1-Pb)*Pa*(c+Fa)
//
// Pa*Fa + Pb*c + Pb*Fb - Pa*Pb*c - Pa*Pb*Fb < Pb*Fb + Pa*c + Pa*Fa - Pa*Pb*c - Pa*Pb*Fa
//
// Removing terms that cancel out:
// Pb*c - Pa*Pb*Fb < Pa*c - Pa*Pb*Fa
//
// Divide by Pa*Pb:
// c/Pa - Fb < c/Pb - Fa
//
// Move terms around:
// Fa + c/Pa < Fb + c/Pb
//
// So the value of F + c/P can be used to compare routes.
func getProbabilityBasedDist(weight int64, probability float64, penalty int64) int64 {
// Clamp probability to prevent overflow.
const minProbability = 0.00001
if probability < minProbability {
return infinity
}
return weight + int64(float64(penalty)/probability)
}

@ -52,7 +52,8 @@ const (
var (
noRestrictions = &RestrictParams{
FeeLimit: noFeeLimit,
FeeLimit: noFeeLimit,
ProbabilitySource: noProbabilitySource,
}
)
@ -72,6 +73,12 @@ var (
}
)
// noProbabilitySource is used in testing to return the same probability 1 for
// all edges.
func noProbabilitySource(route.Vertex, EdgeLocator) float64 {
return 1
}
// testGraph is the struct which corresponds to the JSON format used to encode
// graphs within the files in the testdata directory.
//
@ -635,9 +642,7 @@ func TestFindLowestFeePath(t *testing.T) {
&graphParams{
graph: testGraphInstance.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, paymentAmt,
)
if err != nil {
@ -776,7 +781,8 @@ func testBasicGraphPathFindingCase(t *testing.T, graphInstance *testGraphInstanc
graph: graphInstance.graph,
},
&RestrictParams{
FeeLimit: test.feeLimit,
FeeLimit: test.feeLimit,
ProbabilitySource: noProbabilitySource,
},
sourceNode.PubKeyBytes, target, paymentAmt,
)
@ -944,9 +950,7 @@ func TestPathFindingWithAdditionalEdges(t *testing.T) {
graph: graph.graph,
additionalEdges: additionalEdges,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, doge.PubKeyBytes, paymentAmt,
)
if err != nil {
@ -1200,9 +1204,7 @@ func TestNewRoutePathTooLong(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, paymentAmt,
)
if err != nil {
@ -1216,9 +1218,7 @@ func TestNewRoutePathTooLong(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, paymentAmt,
)
if err == nil {
@ -1258,9 +1258,7 @@ func TestPathNotAvailable(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, unknownNode, 100,
)
if !IsError(err, ErrNoPathFound) {
@ -1297,9 +1295,7 @@ func TestPathInsufficientCapacity(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if !IsError(err, ErrNoPathFound) {
@ -1332,9 +1328,7 @@ func TestRouteFailMinHTLC(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if !IsError(err, ErrNoPathFound) {
@ -1392,9 +1386,7 @@ func TestRouteFailMaxHTLC(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if err != nil {
@ -1416,9 +1408,7 @@ func TestRouteFailMaxHTLC(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if !IsError(err, ErrNoPathFound) {
@ -1453,9 +1443,7 @@ func TestRouteFailDisabledEdge(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if err != nil {
@ -1483,9 +1471,7 @@ func TestRouteFailDisabledEdge(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if err != nil {
@ -1510,9 +1496,7 @@ func TestRouteFailDisabledEdge(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if !IsError(err, ErrNoPathFound) {
@ -1546,9 +1530,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
&graphParams{
graph: graph.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if err != nil {
@ -1572,9 +1554,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
graph: graph.graph,
bandwidthHints: bandwidths,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if !IsError(err, ErrNoPathFound) {
@ -1592,9 +1572,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
graph: graph.graph,
bandwidthHints: bandwidths,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if err != nil {
@ -1625,9 +1603,7 @@ func TestPathSourceEdgesBandwidth(t *testing.T) {
graph: graph.graph,
bandwidthHints: bandwidths,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, payAmt,
)
if err != nil {
@ -1916,6 +1892,7 @@ func TestRestrictOutgoingChannel(t *testing.T) {
&RestrictParams{
FeeLimit: noFeeLimit,
OutgoingChannelID: &outgoingChannelID,
ProbabilitySource: noProbabilitySource,
},
sourceVertex, target, paymentAmt,
)
@ -1992,9 +1969,6 @@ func testCltvLimit(t *testing.T, limit uint32, expectedChannel uint64) {
}
sourceVertex := route.Vertex(sourceNode.PubKeyBytes)
ignoredEdges := make(map[EdgeLocator]struct{})
ignoredVertexes := make(map[route.Vertex]struct{})
paymentAmt := lnwire.NewMSatFromSatoshis(100)
target := testGraphInstance.aliasMap["target"]
@ -2009,10 +1983,9 @@ func testCltvLimit(t *testing.T, limit uint32, expectedChannel uint64) {
graph: testGraphInstance.graph,
},
&RestrictParams{
IgnoredNodes: ignoredVertexes,
IgnoredEdges: ignoredEdges,
FeeLimit: noFeeLimit,
CltvLimit: cltvLimit,
FeeLimit: noFeeLimit,
CltvLimit: cltvLimit,
ProbabilitySource: noProbabilitySource,
},
sourceVertex, target, paymentAmt,
)
@ -2045,3 +2018,134 @@ func testCltvLimit(t *testing.T, limit uint32, expectedChannel uint64) {
route.Hops[0].ChannelID)
}
}
// TestProbabilityRouting asserts that path finding not only takes into account
// fees but also success probability.
func TestProbabilityRouting(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
p10, p11, p20 float64
expectedChan uint64
}{
// Test two variations with probabilities that should multiply
// to the same total route probability. In both cases the three
// hop route should be the best route. The three hop route has a
// probability of 0.5 * 0.8 = 0.4. The fee is 5 (chan 10) + 8
// (chan 11) = 13. Path finding distance should work out to: 13
// + 10 (attempt penalty) / 0.4 = 38. The two hop route is 25 +
// 10 / 0.7 = 39.
{
name: "three hop 1",
p10: 0.8, p11: 0.5, p20: 0.7,
expectedChan: 10,
},
{
name: "three hop 2",
p10: 0.5, p11: 0.8, p20: 0.7,
expectedChan: 10,
},
// If the probability of the two hop route is increased, its
// distance becomes 25 + 10 / 0.85 = 37. This is less than the
// three hop route with its distance 38. So with an attempt
// penalty of 10, the higher fee route is chosen because of the
// compensation for success probability.
{
name: "two hop higher cost",
p10: 0.5, p11: 0.8, p20: 0.85,
expectedChan: 20,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
testProbabilityRouting(
t, tc.p10, tc.p11, tc.p20, tc.expectedChan,
)
})
}
}
func testProbabilityRouting(t *testing.T, p10, p11, p20 float64,
expectedChan uint64) {
t.Parallel()
// Set up a test graph with two possible paths to the target: a three
// hop path (via channels 10 and 11) and a two hop path (via channel
// 20).
testChannels := []*testChannel{
symmetricTestChannel("roasbeef", "a1", 100000, &testChannelPolicy{}),
symmetricTestChannel("roasbeef", "b", 100000, &testChannelPolicy{}),
symmetricTestChannel("roasbeef", "c", 100000, &testChannelPolicy{}),
symmetricTestChannel("a1", "a2", 100000, &testChannelPolicy{
Expiry: 144,
FeeBaseMsat: lnwire.NewMSatFromSatoshis(5),
MinHTLC: 1,
}, 10),
symmetricTestChannel("a2", "target", 100000, &testChannelPolicy{
Expiry: 144,
FeeBaseMsat: lnwire.NewMSatFromSatoshis(8),
MinHTLC: 1,
}, 11),
symmetricTestChannel("b", "target", 100000, &testChannelPolicy{
Expiry: 100,
FeeBaseMsat: lnwire.NewMSatFromSatoshis(25),
MinHTLC: 1,
}, 20),
}
testGraphInstance, err := createTestGraphFromChannels(testChannels)
if err != nil {
t.Fatalf("unable to create graph: %v", err)
}
defer testGraphInstance.cleanUp()
sourceNode, err := testGraphInstance.graph.SourceNode()
if err != nil {
t.Fatalf("unable to fetch source node: %v", err)
}
sourceVertex := route.Vertex(sourceNode.PubKeyBytes)
paymentAmt := lnwire.NewMSatFromSatoshis(100)
target := testGraphInstance.aliasMap["target"]
// Configure a probability source with the test parameters.
probabilitySource := func(node route.Vertex, edge EdgeLocator) float64 {
switch edge.ChannelID {
case 10:
return p10
case 11:
return p11
case 20:
return p20
default:
return 1
}
}
path, err := findPath(
&graphParams{
graph: testGraphInstance.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
ProbabilitySource: probabilitySource,
PaymentAttemptPenalty: lnwire.NewMSatFromSatoshis(10),
},
sourceVertex, target, paymentAmt,
)
if err != nil {
t.Fatal(err)
}
// Assert that the route passes through the expected channel.
if path[1].ChannelID != expectedChan {
t.Fatalf("expected route to pass through channel %v, "+
"but channel %v was selected instead", expectedChan,
path[1].ChannelID)
}
}

@ -143,6 +143,20 @@ func (p *paymentSession) ReportEdgePolicyFailure(
p.errFailedPolicyChans[*failedEdge] = struct{}{}
}
func (p *paymentSession) getEdgeProbability(node route.Vertex,
edge EdgeLocator) float64 {
if _, ok := p.pruneViewSnapshot.vertexes[node]; ok {
return 0
}
if _, ok := p.pruneViewSnapshot.edges[edge]; ok {
return 0
}
return 1
}
// RequestRoute returns a route which is likely to be capable for successfully
// routing the specified HTLC payment to the target node. Initially the first
// set of paths returned from this method may encounter routing failure along
@ -200,11 +214,11 @@ func (p *paymentSession) RequestRoute(payment *LightningPayment,
bandwidthHints: p.bandwidthHints,
},
&RestrictParams{
IgnoredNodes: pruneView.vertexes,
IgnoredEdges: pruneView.edges,
FeeLimit: payment.FeeLimit,
OutgoingChannelID: payment.OutgoingChannelID,
CltvLimit: cltvLimit,
ProbabilitySource: p.getEdgeProbability,
FeeLimit: payment.FeeLimit,
OutgoingChannelID: payment.OutgoingChannelID,
CltvLimit: cltvLimit,
PaymentAttemptPenalty: DefaultPaymentAttemptPenalty,
},
p.mc.selfNode.PubKeyBytes, payment.Target,
payment.Amount,

@ -201,7 +201,8 @@ func TestFindRoutesWithFeeLimit(t *testing.T) {
target := ctx.aliases["sophon"]
paymentAmt := lnwire.NewMSatFromSatoshis(100)
restrictions := &RestrictParams{
FeeLimit: lnwire.NewMSatFromSatoshis(10),
FeeLimit: lnwire.NewMSatFromSatoshis(10),
ProbabilitySource: noProbabilitySource,
}
route, err := ctx.router.FindRoute(
@ -2198,9 +2199,7 @@ func TestFindPathFeeWeighting(t *testing.T) {
&graphParams{
graph: ctx.graph,
},
&RestrictParams{
FeeLimit: noFeeLimit,
},
noRestrictions,
sourceNode.PubKeyBytes, target, amt,
)
if err != nil {