lnd.xprv/lnwallet/chainfee/estimator.go
2020-03-25 14:59:23 +08:00

732 lines
23 KiB
Go

package chainfee
import (
"encoding/json"
"fmt"
"io"
prand "math/rand"
"net"
"net/http"
"sync"
"time"
"github.com/btcsuite/btcd/rpcclient"
"github.com/btcsuite/btcutil"
)
const (
// maxBlockTarget is the highest number of blocks confirmations that
// a WebAPIEstimator will cache fees for. This number is chosen
// because it's the highest number of confs bitcoind will return a fee
// estimate for.
maxBlockTarget uint32 = 1009
// minBlockTarget is the lowest number of blocks confirmations that
// a WebAPIEstimator will cache fees for. Requesting an estimate for
// less than this will result in an error.
minBlockTarget uint32 = 2
// minFeeUpdateTimeout represents the minimum interval in which a
// WebAPIEstimator will request fresh fees from its API.
minFeeUpdateTimeout = 5 * time.Minute
// maxFeeUpdateTimeout represents the maximum interval in which a
// WebAPIEstimator will request fresh fees from its API.
maxFeeUpdateTimeout = 20 * time.Minute
)
// Estimator provides the ability to estimate on-chain transaction fees for
// various combinations of transaction sizes and desired confirmation time
// (measured by number of blocks).
type Estimator interface {
// EstimateFeePerKW takes in a target for the number of blocks until an
// initial confirmation and returns the estimated fee expressed in
// sat/kw.
EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error)
// Start signals the Estimator to start any processes or goroutines
// it needs to perform its duty.
Start() error
// Stop stops any spawned goroutines and cleans up the resources used
// by the fee estimator.
Stop() error
// RelayFeePerKW returns the minimum fee rate required for transactions
// to be relayed. This is also the basis for calculation of the dust
// limit.
RelayFeePerKW() SatPerKWeight
}
// StaticEstimator will return a static value for all fee calculation requests.
// It is designed to be replaced by a proper fee calculation implementation.
// The fees are not accessible directly, because changing them would not be
// thread safe.
type StaticEstimator struct {
// feePerKW is the static fee rate in satoshis-per-vbyte that will be
// returned by this fee estimator.
feePerKW SatPerKWeight
// relayFee is the minimum fee rate required for transactions to be
// relayed.
relayFee SatPerKWeight
}
// NewStaticEstimator returns a new static fee estimator instance.
func NewStaticEstimator(feePerKW, relayFee SatPerKWeight) *StaticEstimator {
return &StaticEstimator{
feePerKW: feePerKW,
relayFee: relayFee,
}
}
// EstimateFeePerKW will return a static value for fee calculations.
//
// NOTE: This method is part of the Estimator interface.
func (e StaticEstimator) EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error) {
return e.feePerKW, nil
}
// RelayFeePerKW returns the minimum fee rate required for transactions to be
// relayed.
//
// NOTE: This method is part of the Estimator interface.
func (e StaticEstimator) RelayFeePerKW() SatPerKWeight {
return e.relayFee
}
// Start signals the Estimator to start any processes or goroutines
// it needs to perform its duty.
//
// NOTE: This method is part of the Estimator interface.
func (e StaticEstimator) Start() error {
return nil
}
// Stop stops any spawned goroutines and cleans up the resources used
// by the fee estimator.
//
// NOTE: This method is part of the Estimator interface.
func (e StaticEstimator) Stop() error {
return nil
}
// A compile-time assertion to ensure that StaticFeeEstimator implements the
// Estimator interface.
var _ Estimator = (*StaticEstimator)(nil)
// BtcdEstimator is an implementation of the Estimator interface backed
// by the RPC interface of an active btcd node. This implementation will proxy
// any fee estimation requests to btcd's RPC interface.
type BtcdEstimator struct {
// fallbackFeePerKW is the fall back fee rate in sat/kw that is returned
// if the fee estimator does not yet have enough data to actually
// produce fee estimates.
fallbackFeePerKW SatPerKWeight
// minFeePerKW is the minimum fee, in sat/kw, that we should enforce.
// This will be used as the default fee rate for a transaction when the
// estimated fee rate is too low to allow the transaction to propagate
// through the network.
minFeePerKW SatPerKWeight
btcdConn *rpcclient.Client
}
// NewBtcdEstimator creates a new BtcdEstimator given a fully populated
// rpc config that is able to successfully connect and authenticate with the
// btcd node, and also a fall back fee rate. The fallback fee rate is used in
// the occasion that the estimator has insufficient data, or returns zero for a
// fee estimate.
func NewBtcdEstimator(rpcConfig rpcclient.ConnConfig,
fallBackFeeRate SatPerKWeight) (*BtcdEstimator, error) {
rpcConfig.DisableConnectOnNew = true
rpcConfig.DisableAutoReconnect = false
chainConn, err := rpcclient.New(&rpcConfig, nil)
if err != nil {
return nil, err
}
return &BtcdEstimator{
fallbackFeePerKW: fallBackFeeRate,
btcdConn: chainConn,
}, nil
}
// Start signals the Estimator to start any processes or goroutines
// it needs to perform its duty.
//
// NOTE: This method is part of the Estimator interface.
func (b *BtcdEstimator) Start() error {
if err := b.btcdConn.Connect(20); err != nil {
return err
}
// Once the connection to the backend node has been established, we'll
// query it for its minimum relay fee.
info, err := b.btcdConn.GetInfo()
if err != nil {
return err
}
relayFee, err := btcutil.NewAmount(info.RelayFee)
if err != nil {
return err
}
// The fee rate is expressed in sat/kb, so we'll manually convert it to
// our desired sat/kw rate.
minRelayFeePerKw := SatPerKVByte(relayFee).FeePerKWeight()
// By default, we'll use the backend node's minimum relay fee as the
// minimum fee rate we'll propose for transacations. However, if this
// happens to be lower than our fee floor, we'll enforce that instead.
b.minFeePerKW = minRelayFeePerKw
if b.minFeePerKW < FeePerKwFloor {
b.minFeePerKW = FeePerKwFloor
}
log.Debugf("Using minimum fee rate of %v sat/kw",
int64(b.minFeePerKW))
return nil
}
// Stop stops any spawned goroutines and cleans up the resources used
// by the fee estimator.
//
// NOTE: This method is part of the Estimator interface.
func (b *BtcdEstimator) Stop() error {
b.btcdConn.Shutdown()
return nil
}
// EstimateFeePerKW takes in a target for the number of blocks until an initial
// confirmation and returns the estimated fee expressed in sat/kw.
//
// NOTE: This method is part of the Estimator interface.
func (b *BtcdEstimator) EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error) {
feeEstimate, err := b.fetchEstimate(numBlocks)
switch {
// If the estimator doesn't have enough data, or returns an error, then
// to return a proper value, then we'll return the default fall back
// fee rate.
case err != nil:
log.Errorf("unable to query estimator: %v", err)
fallthrough
case feeEstimate == 0:
return b.fallbackFeePerKW, nil
}
return feeEstimate, nil
}
// RelayFeePerKW returns the minimum fee rate required for transactions to be
// relayed.
//
// NOTE: This method is part of the Estimator interface.
func (b *BtcdEstimator) RelayFeePerKW() SatPerKWeight {
return b.minFeePerKW
}
// fetchEstimate returns a fee estimate for a transaction to be confirmed in
// confTarget blocks. The estimate is returned in sat/kw.
func (b *BtcdEstimator) fetchEstimate(confTarget uint32) (SatPerKWeight, error) {
// First, we'll fetch the estimate for our confirmation target.
btcPerKB, err := b.btcdConn.EstimateFee(int64(confTarget))
if err != nil {
return 0, err
}
// Next, we'll convert the returned value to satoshis, as it's
// currently returned in BTC.
satPerKB, err := btcutil.NewAmount(btcPerKB)
if err != nil {
return 0, err
}
// Since we use fee rates in sat/kw internally, we'll convert the
// estimated fee rate from its sat/kb representation to sat/kw.
satPerKw := SatPerKVByte(satPerKB).FeePerKWeight()
// Finally, we'll enforce our fee floor.
if satPerKw < b.minFeePerKW {
log.Debugf("Estimated fee rate of %v sat/kw is too low, "+
"using fee floor of %v sat/kw instead", satPerKw,
b.minFeePerKW)
satPerKw = b.minFeePerKW
}
log.Debugf("Returning %v sat/kw for conf target of %v",
int64(satPerKw), confTarget)
return satPerKw, nil
}
// A compile-time assertion to ensure that BtcdEstimator implements the
// Estimator interface.
var _ Estimator = (*BtcdEstimator)(nil)
// BitcoindEstimator is an implementation of the Estimator interface backed by
// the RPC interface of an active bitcoind node. This implementation will proxy
// any fee estimation requests to bitcoind's RPC interface.
type BitcoindEstimator struct {
// fallbackFeePerKW is the fallback fee rate in sat/kw that is returned
// if the fee estimator does not yet have enough data to actually
// produce fee estimates.
fallbackFeePerKW SatPerKWeight
// minFeePerKW is the minimum fee, in sat/kw, that we should enforce.
// This will be used as the default fee rate for a transaction when the
// estimated fee rate is too low to allow the transaction to propagate
// through the network.
minFeePerKW SatPerKWeight
// feeMode is the estimate_mode to use when calling "estimatesmartfee".
// It can be either "ECONOMICAL" or "CONSERVATIVE", and it's default
// to "CONSERVATIVE".
feeMode string
bitcoindConn *rpcclient.Client
}
// NewBitcoindEstimator creates a new BitcoindEstimator given a fully populated
// rpc config that is able to successfully connect and authenticate with the
// bitcoind node, and also a fall back fee rate. The fallback fee rate is used
// in the occasion that the estimator has insufficient data, or returns zero
// for a fee estimate.
func NewBitcoindEstimator(rpcConfig rpcclient.ConnConfig, feeMode string,
fallBackFeeRate SatPerKWeight) (*BitcoindEstimator, error) {
rpcConfig.DisableConnectOnNew = true
rpcConfig.DisableAutoReconnect = false
rpcConfig.DisableTLS = true
rpcConfig.HTTPPostMode = true
chainConn, err := rpcclient.New(&rpcConfig, nil)
if err != nil {
return nil, err
}
return &BitcoindEstimator{
fallbackFeePerKW: fallBackFeeRate,
bitcoindConn: chainConn,
feeMode: feeMode,
}, nil
}
// Start signals the Estimator to start any processes or goroutines
// it needs to perform its duty.
//
// NOTE: This method is part of the Estimator interface.
func (b *BitcoindEstimator) Start() error {
// Once the connection to the backend node has been established, we'll
// query it for its minimum relay fee. Since the `getinfo` RPC has been
// deprecated for `bitcoind`, we'll need to send a `getnetworkinfo`
// command as a raw request.
resp, err := b.bitcoindConn.RawRequest("getnetworkinfo", nil)
if err != nil {
return err
}
// Parse the response to retrieve the relay fee in sat/KB.
info := struct {
RelayFee float64 `json:"relayfee"`
}{}
if err := json.Unmarshal(resp, &info); err != nil {
return err
}
relayFee, err := btcutil.NewAmount(info.RelayFee)
if err != nil {
return err
}
// The fee rate is expressed in sat/kb, so we'll manually convert it to
// our desired sat/kw rate.
minRelayFeePerKw := SatPerKVByte(relayFee).FeePerKWeight()
// By default, we'll use the backend node's minimum relay fee as the
// minimum fee rate we'll propose for transacations. However, if this
// happens to be lower than our fee floor, we'll enforce that instead.
b.minFeePerKW = minRelayFeePerKw
if b.minFeePerKW < FeePerKwFloor {
b.minFeePerKW = FeePerKwFloor
}
log.Debugf("Using minimum fee rate of %v sat/kw",
int64(b.minFeePerKW))
return nil
}
// Stop stops any spawned goroutines and cleans up the resources used
// by the fee estimator.
//
// NOTE: This method is part of the Estimator interface.
func (b *BitcoindEstimator) Stop() error {
return nil
}
// EstimateFeePerKW takes in a target for the number of blocks until an initial
// confirmation and returns the estimated fee expressed in sat/kw.
//
// NOTE: This method is part of the Estimator interface.
func (b *BitcoindEstimator) EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error) {
feeEstimate, err := b.fetchEstimate(numBlocks)
switch {
// If the estimator doesn't have enough data, or returns an error, then
// to return a proper value, then we'll return the default fall back
// fee rate.
case err != nil:
log.Errorf("unable to query estimator: %v", err)
fallthrough
case feeEstimate == 0:
return b.fallbackFeePerKW, nil
}
return feeEstimate, nil
}
// RelayFeePerKW returns the minimum fee rate required for transactions to be
// relayed.
//
// NOTE: This method is part of the Estimator interface.
func (b *BitcoindEstimator) RelayFeePerKW() SatPerKWeight {
return b.minFeePerKW
}
// fetchEstimate returns a fee estimate for a transaction to be confirmed in
// confTarget blocks. The estimate is returned in sat/kw.
func (b *BitcoindEstimator) fetchEstimate(confTarget uint32) (SatPerKWeight, error) {
// First, we'll send an "estimatesmartfee" command as a raw request,
// since it isn't supported by btcd but is available in bitcoind.
target, err := json.Marshal(uint64(confTarget))
if err != nil {
return 0, err
}
// The mode must be either ECONOMICAL or CONSERVATIVE.
mode, err := json.Marshal(b.feeMode)
if err != nil {
return 0, err
}
resp, err := b.bitcoindConn.RawRequest(
"estimatesmartfee", []json.RawMessage{target, mode},
)
if err != nil {
return 0, err
}
// Next, we'll parse the response to get the BTC per KB.
feeEstimate := struct {
FeeRate float64 `json:"feerate"`
}{}
err = json.Unmarshal(resp, &feeEstimate)
if err != nil {
return 0, err
}
// Next, we'll convert the returned value to satoshis, as it's currently
// returned in BTC.
satPerKB, err := btcutil.NewAmount(feeEstimate.FeeRate)
if err != nil {
return 0, err
}
// Since we use fee rates in sat/kw internally, we'll convert the
// estimated fee rate from its sat/kb representation to sat/kw.
satPerKw := SatPerKVByte(satPerKB).FeePerKWeight()
// Finally, we'll enforce our fee floor.
if satPerKw < b.minFeePerKW {
log.Debugf("Estimated fee rate of %v sat/kw is too low, "+
"using fee floor of %v sat/kw instead", satPerKw,
b.minFeePerKW)
satPerKw = b.minFeePerKW
}
log.Debugf("Returning %v sat/kw for conf target of %v",
int64(satPerKw), confTarget)
return satPerKw, nil
}
// A compile-time assertion to ensure that BitcoindEstimator implements the
// Estimator interface.
var _ Estimator = (*BitcoindEstimator)(nil)
// WebAPIFeeSource is an interface allows the WebAPIEstimator to query an
// arbitrary HTTP-based fee estimator. Each new set/network will gain an
// implementation of this interface in order to allow the WebAPIEstimator to
// be fully generic in its logic.
type WebAPIFeeSource interface {
// GenQueryURL generates the full query URL. The value returned by this
// method should be able to be used directly as a path for an HTTP GET
// request.
GenQueryURL() string
// ParseResponse attempts to parse the body of the response generated
// by the above query URL. Typically this will be JSON, but the
// specifics are left to the WebAPIFeeSource implementation.
ParseResponse(r io.Reader) (map[uint32]uint32, error)
}
// SparseConfFeeSource is an implementation of the WebAPIFeeSource that utilizes
// a user-specified fee estimation API for Bitcoin. It expects the response
// to be in the JSON format: `fee_by_block_target: { ... }` where the value maps
// block targets to fee estimates (in sat per kilovbyte).
type SparseConfFeeSource struct {
// URL is the fee estimation API specified by the user.
URL string
}
// GenQueryURL generates the full query URL. The value returned by this
// method should be able to be used directly as a path for an HTTP GET
// request.
//
// NOTE: Part of the WebAPIFeeSource interface.
func (s SparseConfFeeSource) GenQueryURL() string {
return s.URL
}
// ParseResponse attempts to parse the body of the response generated by the
// above query URL. Typically this will be JSON, but the specifics are left to
// the WebAPIFeeSource implementation.
//
// NOTE: Part of the WebAPIFeeSource interface.
func (s SparseConfFeeSource) ParseResponse(r io.Reader) (map[uint32]uint32, error) {
type jsonResp struct {
FeeByBlockTarget map[uint32]uint32 `json:"fee_by_block_target"`
}
resp := jsonResp{
FeeByBlockTarget: make(map[uint32]uint32),
}
jsonReader := json.NewDecoder(r)
if err := jsonReader.Decode(&resp); err != nil {
return nil, err
}
return resp.FeeByBlockTarget, nil
}
// A compile-time assertion to ensure that SparseConfFeeSource implements the
// WebAPIFeeSource interface.
var _ WebAPIFeeSource = (*SparseConfFeeSource)(nil)
// WebAPIEstimator is an implementation of the Estimator interface that
// queries an HTTP-based fee estimation from an existing web API.
type WebAPIEstimator struct {
started sync.Once
stopped sync.Once
// apiSource is the backing web API source we'll use for our queries.
apiSource WebAPIFeeSource
// updateFeeTicker is the ticker responsible for updating the Estimator's
// fee estimates every time it fires.
updateFeeTicker *time.Ticker
// feeByBlockTarget is our cache for fees pulled from the API. When a
// fee estimate request comes in, we pull the estimate from this array
// rather than re-querying the API, to prevent an inadvertent DoS attack.
feesMtx sync.Mutex
feeByBlockTarget map[uint32]uint32
// defaultFeePerKw is a fallback value that we'll use if we're unable
// to query the API for any reason.
defaultFeePerKw SatPerKWeight
quit chan struct{}
wg sync.WaitGroup
}
// NewWebAPIEstimator creates a new WebAPIEstimator from a given URL and a
// fallback default fee. The fees are updated whenever a new block is mined.
func NewWebAPIEstimator(
api WebAPIFeeSource, defaultFee SatPerKWeight) *WebAPIEstimator {
return &WebAPIEstimator{
apiSource: api,
feeByBlockTarget: make(map[uint32]uint32),
defaultFeePerKw: defaultFee,
quit: make(chan struct{}),
}
}
// EstimateFeePerKW takes in a target for the number of blocks until an initial
// confirmation and returns the estimated fee expressed in sat/kw.
//
// NOTE: This method is part of the Estimator interface.
func (w *WebAPIEstimator) EstimateFeePerKW(numBlocks uint32) (SatPerKWeight, error) {
if numBlocks > maxBlockTarget {
numBlocks = maxBlockTarget
} else if numBlocks < minBlockTarget {
return 0, fmt.Errorf("conf target of %v is too low, minimum "+
"accepted is %v", numBlocks, minBlockTarget)
}
feePerKb, err := w.getCachedFee(numBlocks)
if err != nil {
return 0, err
}
// If the result is too low, then we'll clamp it to our current fee
// floor.
satPerKw := SatPerKVByte(feePerKb).FeePerKWeight()
if satPerKw < FeePerKwFloor {
satPerKw = FeePerKwFloor
}
log.Debugf("Web API returning %v sat/kw for conf target of %v",
int64(satPerKw), numBlocks)
return satPerKw, nil
}
// Start signals the Estimator to start any processes or goroutines it needs
// to perform its duty.
//
// NOTE: This method is part of the Estimator interface.
func (w *WebAPIEstimator) Start() error {
var err error
w.started.Do(func() {
log.Infof("Starting web API fee estimator")
w.updateFeeTicker = time.NewTicker(w.randomFeeUpdateTimeout())
w.updateFeeEstimates()
w.wg.Add(1)
go w.feeUpdateManager()
})
return err
}
// Stop stops any spawned goroutines and cleans up the resources used by the
// fee estimator.
//
// NOTE: This method is part of the Estimator interface.
func (w *WebAPIEstimator) Stop() error {
w.stopped.Do(func() {
log.Infof("Stopping web API fee estimator")
w.updateFeeTicker.Stop()
close(w.quit)
w.wg.Wait()
})
return nil
}
// RelayFeePerKW returns the minimum fee rate required for transactions to be
// relayed.
//
// NOTE: This method is part of the Estimator interface.
func (w *WebAPIEstimator) RelayFeePerKW() SatPerKWeight {
return FeePerKwFloor
}
// randomFeeUpdateTimeout returns a random timeout between minFeeUpdateTimeout
// and maxFeeUpdateTimeout that will be used to determine how often the Estimator
// should retrieve fresh fees from its API.
func (w *WebAPIEstimator) randomFeeUpdateTimeout() time.Duration {
lower := int64(minFeeUpdateTimeout)
upper := int64(maxFeeUpdateTimeout)
return time.Duration(prand.Int63n(upper-lower) + lower)
}
// getCachedFee takes in a target for the number of blocks until an initial
// confirmation and returns an estimated fee (if one was returned by the API). If
// the fee was not previously cached, we cache it here.
func (w *WebAPIEstimator) getCachedFee(numBlocks uint32) (uint32, error) {
w.feesMtx.Lock()
defer w.feesMtx.Unlock()
// Search our cached fees for the desired block target. If the target is
// not cached, then attempt to extrapolate it from the next lowest target
// that *is* cached. If we successfully extrapolate, then cache the
// target's fee.
for target := numBlocks; target >= minBlockTarget; target-- {
fee, ok := w.feeByBlockTarget[target]
if !ok {
continue
}
_, ok = w.feeByBlockTarget[numBlocks]
if !ok {
w.feeByBlockTarget[numBlocks] = fee
}
return fee, nil
}
return 0, fmt.Errorf("web API does not include a fee estimation for "+
"block target of %v", numBlocks)
}
// updateFeeEstimates re-queries the API for fresh fees and caches them.
func (w *WebAPIEstimator) updateFeeEstimates() {
// Rather than use the default http.Client, we'll make a custom one
// which will allow us to control how long we'll wait to read the
// response from the service. This way, if the service is down or
// overloaded, we can exit early and use our default fee.
netTransport := &http.Transport{
Dial: (&net.Dialer{
Timeout: 5 * time.Second,
}).Dial,
TLSHandshakeTimeout: 5 * time.Second,
}
netClient := &http.Client{
Timeout: time.Second * 10,
Transport: netTransport,
}
// With the client created, we'll query the API source to fetch the URL
// that we should use to query for the fee estimation.
targetURL := w.apiSource.GenQueryURL()
resp, err := netClient.Get(targetURL)
if err != nil {
log.Errorf("unable to query web api for fee response: %v",
err)
return
}
defer resp.Body.Close()
// Once we've obtained the response, we'll instruct the WebAPIFeeSource
// to parse out the body to obtain our final result.
feesByBlockTarget, err := w.apiSource.ParseResponse(resp.Body)
if err != nil {
log.Errorf("unable to query web api for fee response: %v",
err)
return
}
w.feesMtx.Lock()
w.feeByBlockTarget = feesByBlockTarget
w.feesMtx.Unlock()
}
// feeUpdateManager updates the fee estimates whenever a new block comes in.
func (w *WebAPIEstimator) feeUpdateManager() {
defer w.wg.Done()
for {
select {
case <-w.updateFeeTicker.C:
w.updateFeeEstimates()
case <-w.quit:
return
}
}
}
// A compile-time assertion to ensure that WebAPIEstimator implements the
// Estimator interface.
var _ Estimator = (*WebAPIEstimator)(nil)