From c6b653457bfeb04ab6c8182a1f402e7e0ee6d530 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Wed, 10 Apr 2019 10:12:12 -0700 Subject: [PATCH] lnwallet: add new WebApiFeeEstimator for API fee estimation. Co-authored-by: Valentine Wallace --- lnwallet/fee_estimator.go | 234 +++++++++++++++++++++++++++++++++ lnwallet/fee_estimator_test.go | 90 +++++++++++++ 2 files changed, 324 insertions(+) diff --git a/lnwallet/fee_estimator.go b/lnwallet/fee_estimator.go index f8e76291..65cbef04 100644 --- a/lnwallet/fee_estimator.go +++ b/lnwallet/fee_estimator.go @@ -2,7 +2,13 @@ package lnwallet import ( "encoding/json" + "fmt" "io" + prand "math/rand" + "net" + "net/http" + "sync" + "time" "github.com/btcsuite/btcd/blockchain" "github.com/btcsuite/btcd/rpcclient" @@ -13,6 +19,25 @@ const ( // FeePerKwFloor is the lowest fee rate in sat/kw that we should use for // determining transaction fees. FeePerKwFloor SatPerKWeight = 253 + + // maxBlockTarget is the highest number of blocks confirmations that + // a WebAPIFeeEstimator will cache fees for. This number is chosen + // because it's the highest number of confs bitcoind will return a fee + // estimate for. + maxBlockTarget uint32 = 1009 + + // minBlockTarget is the lowest number of blocks confirmations that + // a WebAPIFeeEstimator will cache fees for. Requesting an estimate for + // less than this will result in an error. + minBlockTarget uint32 = 2 + + // minFeeUpdateTimeout represents the minimum interval in which a + // WebAPIFeeEstimator will request fresh fees from its API. + minFeeUpdateTimeout = 5 * time.Minute + + // maxFeeUpdateTimeout represents the maximum interval in which a + // WebAPIFeeEstimator will request fresh fees from its API. + maxFeeUpdateTimeout = 20 * time.Minute ) // SatPerKVByte represents a fee rate in sat/kb. @@ -518,3 +543,212 @@ func (s SparseConfFeeSource) ParseResponse(r io.Reader) (map[uint32]uint32, erro // A compile-time assertion to ensure that SparseConfFeeSource implements the // WebAPIFeeSource interface. var _ WebAPIFeeSource = (*SparseConfFeeSource)(nil) + +// WebAPIFeeEstimator is an implementation of the FeeEstimator interface that +// queries an HTTP-based fee estimation from an existing web API. +type WebAPIFeeEstimator struct { + started sync.Once + stopped sync.Once + + // apiSource is the backing web API source we'll use for our queries. + apiSource WebAPIFeeSource + + // updateFeeTicker is the ticker responsible for updating the Estimator's + // fee estimates every time it fires. + updateFeeTicker *time.Ticker + + // feeByBlockTarget is our cache for fees pulled from the API. When a + // fee estimate request comes in, we pull the estimate from this array + // rather than re-querying the API, to prevent an inadvertent DoS attack. + feesMtx sync.Mutex + feeByBlockTarget map[uint32]uint32 + + // defaultFeePerKw is a fallback value that we'll use if we're unable + // to query the API for any reason. + defaultFeePerKw SatPerKWeight + + quit chan struct{} + wg sync.WaitGroup +} + +// NewWebAPIFeeEstimator creates a new WebAPIFeeEstimator from a given URL and a +// fallback default fee. The fees are updated whenever a new block is mined. +func NewWebAPIFeeEstimator( + api WebAPIFeeSource, defaultFee SatPerKWeight) *WebAPIFeeEstimator { + + return &WebAPIFeeEstimator{ + apiSource: api, + feeByBlockTarget: make(map[uint32]uint32), + defaultFeePerKw: defaultFee, + quit: make(chan struct{}), + } +} + +// EstimateFeePerKW takes in a target for the number of blocks until an initial +// confirmation and returns the estimated fee expressed in sat/kw. +// +// NOTE: This method is part of the FeeEstimator interface. +func (w *WebAPIFeeEstimator) EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error) { + if numBlocks > maxBlockTarget { + numBlocks = maxBlockTarget + } else if numBlocks < minBlockTarget { + return 0, fmt.Errorf("conf target of %v is too low, minimum "+ + "accepted is %v", numBlocks, minBlockTarget) + } + + feePerKb, err := w.getCachedFee(numBlocks) + if err != nil { + return 0, err + } + + // If the result is too low, then we'll clamp it to our current fee + // floor. + satPerKw := SatPerKVByte(feePerKb).FeePerKWeight() + if satPerKw < FeePerKwFloor { + satPerKw = FeePerKwFloor + } + + walletLog.Debugf("Web API returning %v sat/kw for conf target of %v", + int64(satPerKw), numBlocks) + + return satPerKw, nil +} + +// Start signals the FeeEstimator to start any processes or goroutines it needs +// to perform its duty. +// +// NOTE: This method is part of the FeeEstimator interface. +func (w *WebAPIFeeEstimator) Start() error { + var err error + w.started.Do(func() { + walletLog.Infof("Starting web API fee estimator") + + w.updateFeeTicker = time.NewTicker(w.randomFeeUpdateTimeout()) + w.updateFeeEstimates() + + w.wg.Add(1) + go w.feeUpdateManager() + + }) + return err +} + +// Stop stops any spawned goroutines and cleans up the resources used by the +// fee estimator. +// +// NOTE: This method is part of the FeeEstimator interface. +func (w *WebAPIFeeEstimator) Stop() error { + w.stopped.Do(func() { + walletLog.Infof("Stopping web API fee estimator") + + w.updateFeeTicker.Stop() + + close(w.quit) + w.wg.Wait() + }) + return nil +} + +// RelayFeePerKW returns the minimum fee rate required for transactions to be +// relayed. +// +// NOTE: This method is part of the FeeEstimator interface. +func (w *WebAPIFeeEstimator) RelayFeePerKW() SatPerKWeight { + return FeePerKwFloor +} + +// randomFeeUpdateTimeout returns a random timeout between minFeeUpdateTimeout +// and maxFeeUpdateTimeout that will be used to determine how often the Estimator +// should retrieve fresh fees from its API. +func (w *WebAPIFeeEstimator) randomFeeUpdateTimeout() time.Duration { + lower := int64(minFeeUpdateTimeout) + upper := int64(maxFeeUpdateTimeout) + return time.Duration(prand.Int63n(upper-lower) + lower) +} + +// getCachedFee takes in a target for the number of blocks until an initial +// confirmation and returns an estimated fee (if one was returned by the API). If +// the fee was not previously cached, we cache it here. +func (w *WebAPIFeeEstimator) getCachedFee(numBlocks uint32) (uint32, error) { + w.feesMtx.Lock() + defer w.feesMtx.Unlock() + + // Search our cached fees for the desired block target. If the target is + // not cached, then attempt to extrapolate it from the next lowest target + // that *is* cached. If we successfully extrapolate, then cache the + // target's fee. + for target := numBlocks; target >= minBlockTarget; target-- { + fee, ok := w.feeByBlockTarget[target] + if !ok { + continue + } + + _, ok = w.feeByBlockTarget[numBlocks] + if !ok { + w.feeByBlockTarget[numBlocks] = fee + } + return fee, nil + } + return 0, fmt.Errorf("web API does not include a fee estimation for "+ + "block target of %v", numBlocks) +} + +// updateFeeEstimates re-queries the API for fresh fees and caches them. +func (w *WebAPIFeeEstimator) updateFeeEstimates() { + // Rather than use the default http.Client, we'll make a custom one + // which will allow us to control how long we'll wait to read the + // response from the service. This way, if the service is down or + // overloaded, we can exit early and use our default fee. + netTransport := &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 5 * time.Second, + }).Dial, + TLSHandshakeTimeout: 5 * time.Second, + } + netClient := &http.Client{ + Timeout: time.Second * 10, + Transport: netTransport, + } + + // With the client created, we'll query the API source to fetch the URL + // that we should use to query for the fee estimation. + targetURL := w.apiSource.GenQueryURL() + resp, err := netClient.Get(targetURL) + if err != nil { + walletLog.Errorf("unable to query web api for fee response: %v", + err) + return + } + defer resp.Body.Close() + + // Once we've obtained the response, we'll instruct the WebAPIFeeSource + // to parse out the body to obtain our final result. + feesByBlockTarget, err := w.apiSource.ParseResponse(resp.Body) + if err != nil { + walletLog.Errorf("unable to query web api for fee response: %v", + err) + return + } + + w.feesMtx.Lock() + w.feeByBlockTarget = feesByBlockTarget + w.feesMtx.Unlock() +} + +// feeUpdateManager updates the fee estimates whenever a new block comes in. +func (w *WebAPIFeeEstimator) feeUpdateManager() { + defer w.wg.Done() + + for { + select { + case <-w.updateFeeTicker.C: + w.updateFeeEstimates() + case <-w.quit: + return + } + } +} + +// A compile-time assertion to ensure that WebAPIFeeEstimator implements the +// FeeEstimator interface. +var _ FeeEstimator = (*WebAPIFeeEstimator)(nil) diff --git a/lnwallet/fee_estimator_test.go b/lnwallet/fee_estimator_test.go index d91df345..c2cfeb52 100644 --- a/lnwallet/fee_estimator_test.go +++ b/lnwallet/fee_estimator_test.go @@ -3,13 +3,29 @@ package lnwallet_test import ( "bytes" "encoding/json" + "io" "reflect" + "strings" "testing" "github.com/btcsuite/btcutil" + "github.com/lightningnetwork/lnd/lnwallet" ) +type mockSparseConfFeeSource struct { + url string + fees map[uint32]uint32 +} + +func (e mockSparseConfFeeSource) GenQueryURL() string { + return e.url +} + +func (e mockSparseConfFeeSource) ParseResponse(r io.Reader) (map[uint32]uint32, error) { + return e.fees, nil +} + // TestFeeRateTypes checks that converting fee rates between the // different types that represent fee rates and calculating fees // work as expected. @@ -144,3 +160,77 @@ func TestSparseConfFeeSource(t *testing.T) { t.Fatalf("expected ParseResponse to fail") } } + +// TestWebAPIFeeEstimator checks that the WebAPIFeeEstimator returns fee rates +// as expected. +func TestWebAPIFeeEstimator(t *testing.T) { + t.Parallel() + + feeFloor := uint32(lnwallet.FeePerKwFloor.FeePerKVByte()) + testCases := []struct { + name string + target uint32 + apiEst uint32 + est uint32 + err string + }{ + {"target_below_min", 1, 12345, 12345, "too low, minimum"}, + {"target_w_too-low_fee", 10, 42, feeFloor, ""}, + {"API-omitted_target", 2, 0, 0, "web API does not include"}, + {"valid_target", 20, 54321, 54321, ""}, + {"valid_target_extrapolated_fee", 25, 0, 54321, ""}, + } + + // Construct mock fee source for the Estimator to pull fees from. + testFees := make(map[uint32]uint32) + for _, tc := range testCases { + if tc.apiEst != 0 { + testFees[tc.target] = tc.apiEst + } + } + + feeSource := mockSparseConfFeeSource{ + url: "https://www.github.com", + fees: testFees, + } + + estimator := lnwallet.NewWebAPIFeeEstimator(feeSource, 10) + + // Test that requesting a fee when no fees have been cached fails. + _, err := estimator.EstimateFeePerKW(5) + if err == nil || + !strings.Contains(err.Error(), "web API does not include") { + + t.Fatalf("expected fee estimation to fail, instead got: %v", err) + } + + if err := estimator.Start(); err != nil { + t.Fatalf("unable to start fee estimator, got: %v", err) + } + defer estimator.Stop() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + est, err := estimator.EstimateFeePerKW(tc.target) + if tc.err != "" { + if err == nil || + !strings.Contains(err.Error(), tc.err) { + + t.Fatalf("expected fee estimation to "+ + "fail, instead got: %v", err) + } + } else { + exp := lnwallet.SatPerKVByte(tc.est).FeePerKWeight() + if err != nil { + t.Fatalf("unable to estimate fee for "+ + "%v block target, got: %v", + tc.target, err) + } + if est != exp { + t.Fatalf("expected fee estimate of "+ + "%v, got %v", exp, est) + } + } + }) + } +}