lnwallet: add new WebApiFeeEstimator for API fee estimation.

Co-authored-by: Valentine Wallace <vwallace@protonmail.com>

lnwallet/fee_estimator.go

@@ -2,7 +2,13 @@ package lnwallet
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
+	prand "math/rand"
+	"net"
+	"net/http"
+	"sync"
+	"time"
 
 	"github.com/btcsuite/btcd/blockchain"
 	"github.com/btcsuite/btcd/rpcclient"
@@ -13,6 +19,25 @@ const (
 	// FeePerKwFloor is the lowest fee rate in sat/kw that we should use for
 	// determining transaction fees.
 	FeePerKwFloor SatPerKWeight = 253
+
+	// maxBlockTarget is the highest number of blocks confirmations that
+	// a WebAPIFeeEstimator will cache fees for. This number is chosen
+	// because it's the highest number of confs bitcoind will return a fee
+	// estimate for.
+	maxBlockTarget uint32 = 1009
+
+	// minBlockTarget is the lowest number of blocks confirmations that
+	// a WebAPIFeeEstimator will cache fees for. Requesting an estimate for
+	// less than this will result in an error.
+	minBlockTarget uint32 = 2
+
+	// minFeeUpdateTimeout represents the minimum interval in which a
+	// WebAPIFeeEstimator will request fresh fees from its API.
+	minFeeUpdateTimeout = 5 * time.Minute
+
+	// maxFeeUpdateTimeout represents the maximum interval in which a
+	// WebAPIFeeEstimator will request fresh fees from its API.
+	maxFeeUpdateTimeout = 20 * time.Minute
 )
 
 // SatPerKVByte represents a fee rate in sat/kb.
@@ -518,3 +543,212 @@ func (s SparseConfFeeSource) ParseResponse(r io.Reader) (map[uint32]uint32, erro
 // A compile-time assertion to ensure that SparseConfFeeSource implements the
 // WebAPIFeeSource interface.
 var _ WebAPIFeeSource = (*SparseConfFeeSource)(nil)
+
+// WebAPIFeeEstimator is an implementation of the FeeEstimator interface that
+// queries an HTTP-based fee estimation from an existing web API.
+type WebAPIFeeEstimator struct {
+	started sync.Once
+	stopped sync.Once
+
+	// apiSource is the backing web API source we'll use for our queries.
+	apiSource WebAPIFeeSource
+
+	// updateFeeTicker is the ticker responsible for updating the Estimator's
+	// fee estimates every time it fires.
+	updateFeeTicker *time.Ticker
+
+	// feeByBlockTarget is our cache for fees pulled from the API. When a
+	// fee estimate request comes in, we pull the estimate from this array
+	// rather than re-querying the API, to prevent an inadvertent DoS attack.
+	feesMtx          sync.Mutex
+	feeByBlockTarget map[uint32]uint32
+
+	// defaultFeePerKw is a fallback value that we'll use if we're unable
+	// to query the API for any reason.
+	defaultFeePerKw SatPerKWeight
+
+	quit chan struct{}
+	wg   sync.WaitGroup
+}
+
+// NewWebAPIFeeEstimator creates a new WebAPIFeeEstimator from a given URL and a
+// fallback default fee. The fees are updated whenever a new block is mined.
+func NewWebAPIFeeEstimator(
+	api WebAPIFeeSource, defaultFee SatPerKWeight) *WebAPIFeeEstimator {
+
+	return &WebAPIFeeEstimator{
+		apiSource:        api,
+		feeByBlockTarget: make(map[uint32]uint32),
+		defaultFeePerKw:  defaultFee,
+		quit:             make(chan struct{}),
+	}
+}
+
+// EstimateFeePerKW takes in a target for the number of blocks until an initial
+// confirmation and returns the estimated fee expressed in sat/kw.
+//
+// NOTE: This method is part of the FeeEstimator interface.
+func (w *WebAPIFeeEstimator) EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error) {
+	if numBlocks > maxBlockTarget {
+		numBlocks = maxBlockTarget
+	} else if numBlocks < minBlockTarget {
+		return 0, fmt.Errorf("conf target of %v is too low, minimum "+
+			"accepted is %v", numBlocks, minBlockTarget)
+	}
+
+	feePerKb, err := w.getCachedFee(numBlocks)
+	if err != nil {
+		return 0, err
+	}
+
+	// If the result is too low, then we'll clamp it to our current fee
+	// floor.
+	satPerKw := SatPerKVByte(feePerKb).FeePerKWeight()
+	if satPerKw < FeePerKwFloor {
+		satPerKw = FeePerKwFloor
+	}
+
+	walletLog.Debugf("Web API returning %v sat/kw for conf target of %v",
+		int64(satPerKw), numBlocks)
+
+	return satPerKw, nil
+}
+
+// Start signals the FeeEstimator to start any processes or goroutines it needs
+// to perform its duty.
+//
+// NOTE: This method is part of the FeeEstimator interface.
+func (w *WebAPIFeeEstimator) Start() error {
+	var err error
+	w.started.Do(func() {
+		walletLog.Infof("Starting web API fee estimator")
+
+		w.updateFeeTicker = time.NewTicker(w.randomFeeUpdateTimeout())
+		w.updateFeeEstimates()
+
+		w.wg.Add(1)
+		go w.feeUpdateManager()
+
+	})
+	return err
+}
+
+// Stop stops any spawned goroutines and cleans up the resources used by the
+// fee estimator.
+//
+// NOTE: This method is part of the FeeEstimator interface.
+func (w *WebAPIFeeEstimator) Stop() error {
+	w.stopped.Do(func() {
+		walletLog.Infof("Stopping web API fee estimator")
+
+		w.updateFeeTicker.Stop()
+
+		close(w.quit)
+		w.wg.Wait()
+	})
+	return nil
+}
+
+// RelayFeePerKW returns the minimum fee rate required for transactions to be
+// relayed.
+//
+// NOTE: This method is part of the FeeEstimator interface.
+func (w *WebAPIFeeEstimator) RelayFeePerKW() SatPerKWeight {
+	return FeePerKwFloor
+}
+
+// randomFeeUpdateTimeout returns a random timeout between minFeeUpdateTimeout
+// and maxFeeUpdateTimeout that will be used to determine how often the Estimator
+// should retrieve fresh fees from its API.
+func (w *WebAPIFeeEstimator) randomFeeUpdateTimeout() time.Duration {
+	lower := int64(minFeeUpdateTimeout)
+	upper := int64(maxFeeUpdateTimeout)
+	return time.Duration(prand.Int63n(upper-lower) + lower)
+}
+
+// getCachedFee takes in a target for the number of blocks until an initial
+// confirmation and returns an estimated fee (if one was returned by the API). If
+// the fee was not previously cached, we cache it here.
+func (w *WebAPIFeeEstimator) getCachedFee(numBlocks uint32) (uint32, error) {
+	w.feesMtx.Lock()
+	defer w.feesMtx.Unlock()
+
+	// Search our cached fees for the desired block target. If the target is
+	// not cached, then attempt to extrapolate it from the next lowest target
+	// that *is* cached. If we successfully extrapolate, then cache the
+	// target's fee.
+	for target := numBlocks; target >= minBlockTarget; target-- {
+		fee, ok := w.feeByBlockTarget[target]
+		if !ok {
+			continue
+		}
+
+		_, ok = w.feeByBlockTarget[numBlocks]
+		if !ok {
+			w.feeByBlockTarget[numBlocks] = fee
+		}
+		return fee, nil
+	}
+	return 0, fmt.Errorf("web API does not include a fee estimation for "+
+		"block target of %v", numBlocks)
+}
+
+// updateFeeEstimates re-queries the API for fresh fees and caches them.
+func (w *WebAPIFeeEstimator) updateFeeEstimates() {
+	// Rather than use the default http.Client, we'll make a custom one
+	// which will allow us to control how long we'll wait to read the
+	// response from the service. This way, if the service is down or
+	// overloaded, we can exit early and use our default fee.
+	netTransport := &http.Transport{
+		Dial: (&net.Dialer{
+			Timeout: 5 * time.Second,
+		}).Dial,
+		TLSHandshakeTimeout: 5 * time.Second,
+	}
+	netClient := &http.Client{
+		Timeout:   time.Second * 10,
+		Transport: netTransport,
+	}
+
+	// With the client created, we'll query the API source to fetch the URL
+	// that we should use to query for the fee estimation.
+	targetURL := w.apiSource.GenQueryURL()
+	resp, err := netClient.Get(targetURL)
+	if err != nil {
+		walletLog.Errorf("unable to query web api for fee response: %v",
+			err)
+		return
+	}
+	defer resp.Body.Close()
+
+	// Once we've obtained the response, we'll instruct the WebAPIFeeSource
+	// to parse out the body to obtain our final result.
+	feesByBlockTarget, err := w.apiSource.ParseResponse(resp.Body)
+	if err != nil {
+		walletLog.Errorf("unable to query web api for fee response: %v",
+			err)
+		return
+	}
+
+	w.feesMtx.Lock()
+	w.feeByBlockTarget = feesByBlockTarget
+	w.feesMtx.Unlock()
+}
+
+// feeUpdateManager updates the fee estimates whenever a new block comes in.
+func (w *WebAPIFeeEstimator) feeUpdateManager() {
+	defer w.wg.Done()
+
+	for {
+		select {
+		case <-w.updateFeeTicker.C:
+			w.updateFeeEstimates()
+		case <-w.quit:
+			return
+		}
+	}
+}
+
+// A compile-time assertion to ensure that WebAPIFeeEstimator implements the
+// FeeEstimator interface.
+var _ FeeEstimator = (*WebAPIFeeEstimator)(nil)

lnwallet/fee_estimator_test.go

@@ -3,13 +3,29 @@ package lnwallet_test
 import (
 	"bytes"
 	"encoding/json"
+	"io"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/btcsuite/btcutil"
+
 	"github.com/lightningnetwork/lnd/lnwallet"
 )
 
+type mockSparseConfFeeSource struct {
+	url  string
+	fees map[uint32]uint32
+}
+
+func (e mockSparseConfFeeSource) GenQueryURL() string {
+	return e.url
+}
+
+func (e mockSparseConfFeeSource) ParseResponse(r io.Reader) (map[uint32]uint32, error) {
+	return e.fees, nil
+}
+
 // TestFeeRateTypes checks that converting fee rates between the
 // different types that represent fee rates and calculating fees
 // work as expected.
@@ -144,3 +160,77 @@ func TestSparseConfFeeSource(t *testing.T) {
 		t.Fatalf("expected ParseResponse to fail")
 	}
 }
+
+// TestWebAPIFeeEstimator checks that the WebAPIFeeEstimator returns fee rates
+// as expected.
+func TestWebAPIFeeEstimator(t *testing.T) {
+	t.Parallel()
+
+	feeFloor := uint32(lnwallet.FeePerKwFloor.FeePerKVByte())
+	testCases := []struct {
+		name   string
+		target uint32
+		apiEst uint32
+		est    uint32
+		err    string
+	}{
+		{"target_below_min", 1, 12345, 12345, "too low, minimum"},
+		{"target_w_too-low_fee", 10, 42, feeFloor, ""},
+		{"API-omitted_target", 2, 0, 0, "web API does not include"},
+		{"valid_target", 20, 54321, 54321, ""},
+		{"valid_target_extrapolated_fee", 25, 0, 54321, ""},
+	}
+
+	// Construct mock fee source for the Estimator to pull fees from.
+	testFees := make(map[uint32]uint32)
+	for _, tc := range testCases {
+		if tc.apiEst != 0 {
+			testFees[tc.target] = tc.apiEst
+		}
+	}
+
+	feeSource := mockSparseConfFeeSource{
+		url:  "https://www.github.com",
+		fees: testFees,
+	}
+
+	estimator := lnwallet.NewWebAPIFeeEstimator(feeSource, 10)
+
+	// Test that requesting a fee when no fees have been cached fails.
+	_, err := estimator.EstimateFeePerKW(5)
+	if err == nil ||
+		!strings.Contains(err.Error(), "web API does not include") {
+
+		t.Fatalf("expected fee estimation to fail, instead got: %v", err)
+	}
+
+	if err := estimator.Start(); err != nil {
+		t.Fatalf("unable to start fee estimator, got: %v", err)
+	}
+	defer estimator.Stop()
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			est, err := estimator.EstimateFeePerKW(tc.target)
+			if tc.err != "" {
+				if err == nil ||
+					!strings.Contains(err.Error(), tc.err) {
+
+					t.Fatalf("expected fee estimation to "+
+						"fail, instead got: %v", err)
+				}
+			} else {
+				exp := lnwallet.SatPerKVByte(tc.est).FeePerKWeight()
+				if err != nil {
+					t.Fatalf("unable to estimate fee for "+
+						"%v block target, got: %v",
+						tc.target, err)
+				}
+				if est != exp {
+					t.Fatalf("expected fee estimate of "+
+						"%v, got %v", exp, est)
+				}
+			}
+		})
+	}
+}