Merge branch 'master' into enable-encode-invoice-via-rpc

This commit is contained in:
Yaacov Akiba Slama committed 2020-11-04 16:47:01 +02:00
commit 12f8cdd6d5
258 files changed
+17089 -7125

No files matched your search

+1
View File
@@ -214,6 +214,7 @@ jobs:
matrix:
unit_type:
- btcd unit-cover
- unit tags=kvdb_etcd
- travis-race
steps:
- name: git checkout
+9 -1
View File
@@ -1,6 +1,6 @@
run:
# timeout for analysis
deadline: 4m
deadline: 10m
# Skip autogenerated files for mobile.
skip-files:
@@ -65,3 +65,11 @@ issues:
- path: _test\.go
linters:
- gosec
# Fix false positives because of build flags in itest directory.
- path: lntest/itest/.*
linters:
- unused
- deadcode
- unparam
- govet
+48 -4
View File
@@ -28,6 +28,25 @@ sudo: required
jobs:
include:
- stage: Sanity Check
name: Lint and compile
before_script:
# Install the RPC tools as a before step so Travis collapses the output
# after it's done.
- ./scripts/install_travis_proto.sh
script:
# Step 1: Make sure no diff is produced when compiling with the correct
# version.
- make rpc-check
# Step 2: Make sure the unit tests compile, but don't run them. They run
# in a GitHub Workflow.
- make unit pkg=... case=_NONE_
# Step 3: Lint go code. Invoke GC more often to reduce memory usage.
- GOGC=30 make lint
- stage: Integration Test
name: Btcd Integration
script:
@@ -49,7 +68,32 @@ jobs:
- GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make itest-only
arch: arm64
after_script:
- LOG_FILES=./lntest/itest/*.log
- echo "Uploading to termbin.com..." && find $LOG_FILES | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
- echo "Uploading to file.io..." && tar -zcvO $LOG_FILES | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'
- name: Btcd Integration Windows
script:
- make itest-windows
os: windows
before_install:
- choco upgrade --no-progress -y make netcat curl findutils
- export MAKE=mingw32-make
after_failure:
- |-
case $TRAVIS_OS_NAME in
windows)
echo "Uploading to termbin.com..."
for f in ./lntest/itest/*.log; do cat $f | nc termbin.com 9999 | xargs -r0 printf "$f"' uploaded to %s'; done
;;
esac
after_failure:
- |-
case $TRAVIS_OS_NAME in
windows)
# Needs other commands, see after_script of the Windows build
;;
*)
LOG_FILES=./lntest/itest/*.log
echo "Uploading to termbin.com..." && find $LOG_FILES | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
echo "Uploading to file.io..." && tar -zcvO $LOG_FILES | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'
;;
esac
+7 -2
View File
@@ -43,8 +43,6 @@ GOTEST := GO111MODULE=on go test
GOVERSION := $(shell go version | awk '{print $$3}')
GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
GOLIST := go list -deps $(PKG)/... | grep '$(PKG)'| grep -v '/vendor/'
GOLISTCOVER := $(shell go list -deps -f '{{.ImportPath}}' ./... | grep '$(PKG)' | sed -e 's/^$(ESCPKG)/./')
RM := rm -f
CP := cp
@@ -132,6 +130,11 @@ build-itest:
$(GOBUILD) -tags="$(ITEST_TAGS)" -o lnd-itest $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
$(GOBUILD) -tags="$(ITEST_TAGS)" -o lncli-itest $(ITEST_LDFLAGS) $(PKG)/cmd/lncli
build-itest-windows:
@$(call print, "Building itest lnd and lncli.")
$(GOBUILD) -tags="$(ITEST_TAGS)" -o lnd-itest.exe $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
$(GOBUILD) -tags="$(ITEST_TAGS)" -o lncli-itest.exe $(ITEST_LDFLAGS) $(PKG)/cmd/lncli
install:
@$(call print, "Installing lnd and lncli.")
$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/lnd
@@ -158,6 +161,8 @@ itest-only:
itest: btcd build-itest itest-only
itest-windows: btcd build-itest-windows itest-only
unit: btcd
@$(call print, "Running unit tests.")
$(UNIT)
+1 -1
View File
@@ -90,7 +90,7 @@ any issues regarding security or privacy, please disclose the information
responsibly by sending an email to security at lightning dot engineering,
preferably [encrypted using our designated PGP key
(`91FE464CD75101DA6B6BAB60555C6465E5BCB3AF`) which can be found
[here](https://gist.githubusercontent.com/Roasbeef/6fb5b52886183239e4aa558f83d085d3/raw/5ef96c426e3cf20a2443dc9d3c7d6877576da9ca/security@lightning.engineering).
[here](https://gist.githubusercontent.com/Roasbeef/6fb5b52886183239e4aa558f83d085d3/raw/5fa96010af201628bcfa61e9309d9b13d23d220f/security@lightning.engineering).
## Further reading
* [Step-by-step send payment guide with docker](https://github.com/lightningnetwork/lnd/tree/master/docker)
+8 -1
View File
@@ -20,6 +20,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/htlcswitch"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/labels"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
)
@@ -566,7 +567,8 @@ justiceTxBroadcast:
// We'll now attempt to broadcast the transaction which finalized the
// channel's retribution against the cheating counter party.
err = b.cfg.PublishTransaction(finalTx, "")
label := labels.MakeLabel(labels.LabelTypeJusticeTransaction, nil)
err = b.cfg.PublishTransaction(finalTx, label)
if err != nil {
brarLog.Errorf("Unable to broadcast justice tx: %v", err)
@@ -912,6 +914,11 @@ func (bo *breachedOutput) HeightHint() uint32 {
return bo.confHeight
}
// UnconfParent returns information about a possibly unconfirmed parent tx.
func (bo *breachedOutput) UnconfParent() *input.TxInfo {
return nil
}
// Add compile-time constraint ensuring breachedOutput implements the Input
// interface.
var _ input.Input = (*breachedOutput)(nil)
+8 -7
View File
@@ -29,6 +29,7 @@ import (
"github.com/lightningnetwork/lnd/htlcswitch"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/keychain"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lntest/wait"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
@@ -1433,8 +1434,8 @@ func testBreachSpends(t *testing.T, test breachTest) {
// Notify that the breaching transaction is confirmed, to trigger the
// retribution logic.
notifier := brar.cfg.Notifier.(*mockSpendNotifier)
notifier.confChannel <- &chainntnfs.TxConfirmation{}
notifier := brar.cfg.Notifier.(*mock.SpendNotifier)
notifier.ConfChan <- &chainntnfs.TxConfirmation{}
// The breach arbiter should attempt to sweep all outputs on the
// breached commitment. We'll pretend that the HTLC output has been
@@ -1510,7 +1511,7 @@ func testBreachSpends(t *testing.T, test breachTest) {
// Deliver confirmation of sweep if the test expects it.
if test.sendFinalConf {
notifier.confChannel <- &chainntnfs.TxConfirmation{}
notifier.ConfChan <- &chainntnfs.TxConfirmation{}
}
// Assert that the channel is fully resolved.
@@ -1670,10 +1671,10 @@ func createTestArbiter(t *testing.T, contractBreaches chan *ContractBreachEvent,
aliceKeyPriv, _ := btcec.PrivKeyFromBytes(btcec.S256(),
alicesPrivKey)
signer := &mockSigner{key: aliceKeyPriv}
signer := &mock.SingleSigner{Privkey: aliceKeyPriv}
// Assemble our test arbiter.
notifier := makeMockSpendNotifier()
notifier := mock.MakeMockSpendNotifier()
ba := newBreachArbiter(&BreachConfig{
CloseLink: func(_ *wire.OutPoint, _ htlcswitch.ChannelCloseType) {},
DB: db,
@@ -1897,8 +1898,8 @@ func createInitChannels(revocationWindow int) (*lnwallet.LightningChannel, *lnwa
Packager: channeldb.NewChannelPackager(shortChanID),
}
aliceSigner := &mockSigner{aliceKeyPriv}
bobSigner := &mockSigner{bobKeyPriv}
aliceSigner := &mock.SingleSigner{Privkey: aliceKeyPriv}
bobSigner := &mock.SingleSigner{Privkey: bobKeyPriv}
alicePool := lnwallet.NewSigPool(1, aliceSigner)
channelAlice, err := lnwallet.NewLightningChannel(
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/btcsuite/btcd/btcec"
"github.com/lightningnetwork/lnd/keychain"
"github.com/lightningnetwork/lnd/lnwire"
"github.com/lightningnetwork/lnd/tor"
)
// Conn is an implementation of net.Conn which enforces an authenticated key
@@ -34,12 +35,12 @@ var _ net.Conn = (*Conn)(nil)
// public key. In the case of a handshake failure, the connection is closed and
// a non-nil error is returned.
func Dial(local keychain.SingleKeyECDH, netAddr *lnwire.NetAddress,
dialer func(string, string) (net.Conn, error)) (*Conn, error) {
timeout time.Duration, dialer tor.DialFunc) (*Conn, error) {
ipAddr := netAddr.Address.String()
var conn net.Conn
var err error
conn, err = dialer("tcp", ipAddr)
conn, err = dialer("tcp", ipAddr, timeout)
if err != nil {
return nil, err
}
+9 -2
View File
@@ -13,6 +13,7 @@ import (
"github.com/btcsuite/btcd/btcec"
"github.com/lightningnetwork/lnd/keychain"
"github.com/lightningnetwork/lnd/lnwire"
"github.com/lightningnetwork/lnd/tor"
)
type maybeNetConn struct {
@@ -66,7 +67,10 @@ func establishTestConnection() (net.Conn, net.Conn, func(), error) {
// successful.
remoteConnChan := make(chan maybeNetConn, 1)
go func() {
remoteConn, err := Dial(remoteKeyECDH, netAddr, net.Dial)
remoteConn, err := Dial(
remoteKeyECDH, netAddr,
tor.DefaultConnTimeout, net.DialTimeout,
)
remoteConnChan <- maybeNetConn{remoteConn, err}
}()
@@ -196,7 +200,10 @@ func TestConcurrentHandshakes(t *testing.T) {
remoteKeyECDH := &keychain.PrivKeyECDH{PrivKey: remotePriv}
go func() {
remoteConn, err := Dial(remoteKeyECDH, netAddr, net.Dial)
remoteConn, err := Dial(
remoteKeyECDH, netAddr,
tor.DefaultConnTimeout, net.DialTimeout,
)
connChan <- maybeNetConn{remoteConn, err}
}()
+53
View File
@@ -0,0 +1,53 @@
package build
import (
"github.com/btcsuite/btclog"
"github.com/lightningnetwork/lnd/signal"
)
// ShutdownLogger wraps an existing logger with a shutdown function which will
// be called on Critical/Criticalf to prompt shutdown.
type ShutdownLogger struct {
btclog.Logger
}
// NewShutdownLogger creates a shutdown logger for the log provided which will
// use the signal package to request shutdown on critical errors.
func NewShutdownLogger(logger btclog.Logger) *ShutdownLogger {
return &ShutdownLogger{
Logger: logger,
}
}
// Criticalf formats message according to format specifier and writes to
// log with LevelCritical. It will then call the shutdown logger's shutdown
// function to prompt safe shutdown.
//
// Note: it is part of the btclog.Logger interface.
func (s *ShutdownLogger) Criticalf(format string, params ...interface{}) {
s.Logger.Criticalf(format, params...)
s.shutdown()
}
// Critical formats message using the default formats for its operands
// and writes to log with LevelCritical. It will then call the shutdown
// logger's shutdown function to prompt safe shutdown.
//
// Note: it is part of the btclog.Logger interface.
func (s *ShutdownLogger) Critical(v ...interface{}) {
s.Logger.Critical(v)
s.shutdown()
}
// shutdown checks whether we are listening for interrupts, since a shutdown
// request to the signal package will block if it is not running, and requests
// shutdown if possible.
func (s *ShutdownLogger) shutdown() {
if !signal.Listening() {
s.Logger.Info("Request for shutdown ignored")
return
}
s.Logger.Info("Sending request for shutdown")
signal.RequestShutdown()
}
+4 -1
View File
@@ -39,7 +39,10 @@ func NewRotatingLogWriter() *RotatingLogWriter {
logWriter := &LogWriter{}
backendLog := btclog.NewBackend(logWriter)
return &RotatingLogWriter{
GenSubLogger: backendLog.Logger,
GenSubLogger: func(tag string) btclog.Logger {
logger := backendLog.Logger(tag)
return NewShutdownLogger(logger)
},
logWriter: logWriter,
backendLog: backendLog,
subsystemLoggers: SubLoggers{},
+1 -1
View File
@@ -41,7 +41,7 @@ const (
AppMajor uint = 0
// AppMinor defines the minor version of this binary.
AppMinor uint = 10
AppMinor uint = 11
// AppPatch defines the application patch for this binary.
AppPatch uint = 99
+2
View File
@@ -1,3 +1,5 @@
module github.com/lightningnetwork/lnd/cert
go 1.13
require github.com/stretchr/testify v1.5.1
+11
View File
@@ -0,0 +1,11 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+37 -19
View File
@@ -32,8 +32,9 @@ var (
)
// ipAddresses returns the parserd IP addresses to use when creating the TLS
// certificate.
func ipAddresses(tlsExtraIPs []string) ([]net.IP, error) {
// certificate. If tlsDisableAutofill is true, we don't include interface
// addresses to protect users privacy.
func ipAddresses(tlsExtraIPs []string, tlsDisableAutofill bool) ([]net.IP, error) {
// Collect the host's IP addresses, including loopback, in a slice.
ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
@@ -47,15 +48,20 @@ func ipAddresses(tlsExtraIPs []string) ([]net.IP, error) {
ipAddresses = append(ipAddresses, ipAddr)
}
// Add all the interface IPs that aren't already in the slice.
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, err
}
for _, a := range addrs {
ipAddr, _, err := net.ParseCIDR(a.String())
if err == nil {
addIP(ipAddr)
// To protect their privacy, some users might not want to have all
// their network addresses include in the certificate as this could
// leak sensitive information.
if !tlsDisableAutofill {
// Add all the interface IPs that aren't already in the slice.
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil, err
}
for _, a := range addrs {
ipAddr, _, err := net.ParseCIDR(a.String())
if err == nil {
addIP(ipAddr)
}
}
}
@@ -72,10 +78,14 @@ func ipAddresses(tlsExtraIPs []string) ([]net.IP, error) {
// dnsNames returns the host and DNS names to use when creating the TLS
// ceftificate.
func dnsNames(tlsExtraDomains []string) (string, []string) {
func dnsNames(tlsExtraDomains []string, tlsDisableAutofill bool) (string, []string) {
// Collect the host's names into a slice.
host, err := os.Hostname()
if err != nil {
// To further protect their privacy, some users might not want
// to have their hostname include in the certificate as this could
// leak sensitive information.
if err != nil || tlsDisableAutofill {
// Nothing much we can do here, other than falling back to
// localhost as fallback. A hostname can still be provided with
// the tlsExtraDomain parameter if the problem persists on a
@@ -89,6 +99,13 @@ func dnsNames(tlsExtraDomains []string) (string, []string) {
}
dnsNames = append(dnsNames, tlsExtraDomains...)
// Because we aren't including the hostname in the certificate when
// tlsDisableAutofill is set, we will use the first extra domain
// specified by the user, if it's set, as the Common Name.
if tlsDisableAutofill && len(tlsExtraDomains) > 0 {
host = tlsExtraDomains[0]
}
// Also add fake hostnames for unix sockets, otherwise hostname
// verification will fail in the client.
dnsNames = append(dnsNames, "unix", "unixpacket")
@@ -104,10 +121,10 @@ func dnsNames(tlsExtraDomains []string) (string, []string) {
// and domains given. The certificate is considered up to date if it was
// created with _exactly_ the IPs and domains given.
func IsOutdated(cert *x509.Certificate, tlsExtraIPs,
tlsExtraDomains []string) (bool, error) {
tlsExtraDomains []string, tlsDisableAutofill bool) (bool, error) {
// Parse the slice of IP strings.
ips, err := ipAddresses(tlsExtraIPs)
ips, err := ipAddresses(tlsExtraIPs, tlsDisableAutofill)
if err != nil {
return false, err
}
@@ -147,7 +164,7 @@ func IsOutdated(cert *x509.Certificate, tlsExtraIPs,
}
// Get the full list of DNS names to use.
_, dnsNames := dnsNames(tlsExtraDomains)
_, dnsNames := dnsNames(tlsExtraDomains, tlsDisableAutofill)
// We do the same kind of deduplication for the DNS names.
dns1 := make(map[string]struct{})
@@ -186,7 +203,8 @@ func IsOutdated(cert *x509.Certificate, tlsExtraIPs,
// This function is adapted from https://github.com/btcsuite/btcd and
// https://github.com/btcsuite/btcutil
func GenCertPair(org, certFile, keyFile string, tlsExtraIPs,
tlsExtraDomains []string, certValidity time.Duration) error {
tlsExtraDomains []string, tlsDisableAutofill bool,
certValidity time.Duration) error {
now := time.Now()
validUntil := now.Add(certValidity)
@@ -204,8 +222,8 @@ func GenCertPair(org, certFile, keyFile string, tlsExtraIPs,
// Get all DNS names and IP addresses to use when creating the
// certificate.
host, dnsNames := dnsNames(tlsExtraDomains)
ipAddresses, err := ipAddresses(tlsExtraIPs)
host, dnsNames := dnsNames(tlsExtraDomains, tlsDisableAutofill)
ipAddresses, err := ipAddresses(tlsExtraIPs, tlsDisableAutofill)
if err != nil {
return err
}
+62 -5
View File
@@ -5,6 +5,7 @@ import (
"testing"
"github.com/lightningnetwork/lnd/cert"
"github.com/stretchr/testify/require"
)
var (
@@ -26,7 +27,7 @@ func TestIsOutdatedCert(t *testing.T) {
// Generate TLS files with two extra IPs and domains.
err = cert.GenCertPair(
"lnd autogenerated cert", certPath, keyPath, extraIPs[:2],
extraDomains[:2], cert.DefaultAutogenValidity,
extraDomains[:2], false, cert.DefaultAutogenValidity,
)
if err != nil {
t.Fatal(err)
@@ -48,7 +49,7 @@ func TestIsOutdatedCert(t *testing.T) {
// above.
outdated, err := cert.IsOutdated(
parsedCert, extraIPs[:numIPs],
extraDomains[:numDomains],
extraDomains[:numDomains], false,
)
if err != nil {
t.Fatal(err)
@@ -81,7 +82,7 @@ func TestIsOutdatedPermutation(t *testing.T) {
// Generate TLS files from the IPs and domains.
err = cert.GenCertPair(
"lnd autogenerated cert", certPath, keyPath, extraIPs[:],
extraDomains[:], cert.DefaultAutogenValidity,
extraDomains[:], false, cert.DefaultAutogenValidity,
)
if err != nil {
t.Fatal(err)
@@ -102,7 +103,7 @@ func TestIsOutdatedPermutation(t *testing.T) {
dupDNS[i] = extraDomains[i/2]
}
outdated, err := cert.IsOutdated(parsedCert, dupIPs, dupDNS)
outdated, err := cert.IsOutdated(parsedCert, dupIPs, dupDNS, false)
if err != nil {
t.Fatal(err)
}
@@ -123,7 +124,7 @@ func TestIsOutdatedPermutation(t *testing.T) {
revDNS[i] = extraDomains[len(extraDomains)-1-i]
}
outdated, err = cert.IsOutdated(parsedCert, revIPs, revDNS)
outdated, err = cert.IsOutdated(parsedCert, revIPs, revDNS, false)
if err != nil {
t.Fatal(err)
}
@@ -133,3 +134,59 @@ func TestIsOutdatedPermutation(t *testing.T) {
"considered outdated")
}
}
// TestTLSDisableAutofill checks that setting the --tlsdisableautofill flag
// does not add interface ip addresses or hostnames to the cert.
func TestTLSDisableAutofill(t *testing.T) {
tempDir, err := ioutil.TempDir("", "certtest")
if err != nil {
t.Fatal(err)
}
certPath := tempDir + "/tls.cert"
keyPath := tempDir + "/tls.key"
// Generate TLS files with two extra IPs and domains and no interface IPs.
err = cert.GenCertPair(
"lnd autogenerated cert", certPath, keyPath, extraIPs[:2],
extraDomains[:2], true, cert.DefaultAutogenValidity,
)
require.NoError(
t, err,
"unable to generate tls certificate pair",
)
_, parsedCert, err := cert.LoadCert(
certPath, keyPath,
)
require.NoError(
t, err,
"unable to load tls certificate pair",
)
// Check if the TLS cert is outdated while still preventing
// interface IPs from being used. Should not be outdated
shouldNotBeOutdated, err := cert.IsOutdated(
parsedCert, extraIPs[:2],
extraDomains[:2], true,
)
require.NoError(t, err)
require.Equal(
t, false, shouldNotBeOutdated,
"TLS Certificate was marked as outdated when it should not be",
)
// Check if the TLS cert is outdated while allowing for
// interface IPs to be used. Should report as outdated.
shouldBeOutdated, err := cert.IsOutdated(
parsedCert, extraIPs[:2],
extraDomains[:2], false,
)
require.NoError(t, err)
require.Equal(
t, true, shouldBeOutdated,
"TLS Certificate was not marked as outdated when it should be",
)
}
+1 -1
View File
@@ -482,7 +482,7 @@ func GetClientMissedBlocks(chainConn ChainConn, clientBestBlock *BlockEpoch,
return missedBlocks, nil
}
// RewindChain handles internal state updates for the notifier's TxNotifier It
// RewindChain handles internal state updates for the notifier's TxNotifier. It
// has no effect if given a height greater than or equal to our current best
// known height. It returns the new best block for the notifier.
func RewindChain(chainConn ChainConn, txNotifier *TxNotifier,
+6 -6
View File
@@ -262,7 +262,7 @@ type HistoricalConfDispatch struct {
// inclusion of within the chain.
ConfRequest
// StartHeight specifies the block height at which to being the
// StartHeight specifies the block height at which to begin the
// historical rescan.
StartHeight uint32
@@ -763,9 +763,7 @@ func (n *TxNotifier) CancelConf(confRequest ConfRequest, confID uint64) {
if confSet.details != nil {
confHeight := confSet.details.BlockHeight +
ntfn.NumConfirmations - 1
if confHeight <= n.currentHeight {
delete(n.ntfnsByConfirmHeight[confHeight], ntfn)
}
delete(n.ntfnsByConfirmHeight[confHeight], ntfn)
}
}
@@ -1959,19 +1957,21 @@ func (n *TxNotifier) TearDown() {
defer n.Unlock()
for _, confSet := range n.confNotifications {
for _, ntfn := range confSet.ntfns {
for confID, ntfn := range confSet.ntfns {
close(ntfn.Event.Confirmed)
close(ntfn.Event.Updates)
close(ntfn.Event.NegativeConf)
close(ntfn.Event.Done)
delete(confSet.ntfns, confID)
}
}
for _, spendSet := range n.spendNotifications {
for _, ntfn := range spendSet.ntfns {
for spendID, ntfn := range spendSet.ntfns {
close(ntfn.Event.Spend)
close(ntfn.Event.Reorg)
close(ntfn.Event.Done)
delete(spendSet.ntfns, spendID)
}
}
}
+58 -4
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"sync"
"testing"
"time"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/wire"
@@ -1138,7 +1139,7 @@ func TestTxNotifierCancelConf(t *testing.T) {
hintCache := newMockHintCache()
n := chainntnfs.NewTxNotifier(startingHeight, 100, hintCache, hintCache)
// We'll register three notification requests. The last two will be
// We'll register four notification requests. The last three will be
// canceled.
tx1 := wire.NewMsgTx(1)
tx1.AddTxOut(&wire.TxOut{PkScript: testRawScript})
@@ -1159,7 +1160,12 @@ func TestTxNotifierCancelConf(t *testing.T) {
if err != nil {
t.Fatalf("unable to register spend ntfn: %v", err)
}
cancelConfRequest := ntfn2.HistoricalDispatch.ConfRequest
// This request will have a three block num confs.
ntfn4, err := n.RegisterConf(&tx2Hash, testRawScript, 3, 1)
if err != nil {
t.Fatalf("unable to register spend ntfn: %v", err)
}
// Extend the chain with a block that will confirm both transactions.
// This will queue confirmation notifications to dispatch once their
@@ -1175,7 +1181,7 @@ func TestTxNotifierCancelConf(t *testing.T) {
}
// Cancel the second notification before connecting the block.
n.CancelConf(cancelConfRequest, 2)
ntfn2.Event.Cancel()
err = n.ConnectTip(block.Hash(), startingHeight+1, block.Transactions())
if err != nil {
@@ -1184,7 +1190,7 @@ func TestTxNotifierCancelConf(t *testing.T) {
// Cancel the third notification before notifying to ensure its queued
// confirmation notification gets removed as well.
n.CancelConf(cancelConfRequest, 3)
ntfn3.Event.Cancel()
if err := n.NotifyHeight(startingHeight + 1); err != nil {
t.Fatalf("unable to dispatch notifications: %v", err)
@@ -1218,6 +1224,54 @@ func TestTxNotifierCancelConf(t *testing.T) {
default:
t.Fatal("expected Confirmed channel to be closed")
}
// Connect yet another block.
block1 := btcutil.NewBlock(&wire.MsgBlock{
Transactions: []*wire.MsgTx{},
})
err = n.ConnectTip(block1.Hash(), startingHeight+2, block1.Transactions())
if err != nil {
t.Fatalf("unable to connect block: %v", err)
}
if err := n.NotifyHeight(startingHeight + 2); err != nil {
t.Fatalf("unable to dispatch notifications: %v", err)
}
// Since neither it reached the set confirmation height or was
// canceled, nothing should happen to ntfn4 in this block.
select {
case <-ntfn4.Event.Confirmed:
t.Fatal("expected nothing to happen")
case <-time.After(10 * time.Millisecond):
}
// Now cancel the notification.
ntfn4.Event.Cancel()
select {
case _, ok := <-ntfn4.Event.Confirmed:
if ok {
t.Fatal("expected Confirmed channel to be closed")
}
default:
t.Fatal("expected Confirmed channel to be closed")
}
// Finally, confirm a block that would trigger ntfn4 confirmation
// hadn't it already been canceled.
block2 := btcutil.NewBlock(&wire.MsgBlock{
Transactions: []*wire.MsgTx{},
})
err = n.ConnectTip(block2.Hash(), startingHeight+3, block2.Transactions())
if err != nil {
t.Fatalf("unable to connect block: %v", err)
}
if err := n.NotifyHeight(startingHeight + 3); err != nil {
t.Fatalf("unable to dispatch notifications: %v", err)
}
}
// TestTxNotifierCancelSpend ensures that a spend notification after a client
-4
View File
@@ -10,10 +10,6 @@ import (
litecoinWire "github.com/ltcsuite/ltcd/wire"
)
// activeNetParams is a pointer to the parameters specific to the currently
// active bitcoin network.
var activeNetParams = bitcoinTestNetParams
// bitcoinNetParams couples the p2p parameters of a network with the
// corresponding RPC port of a daemon running on the particular network.
type bitcoinNetParams struct {
+51 -39
View File
@@ -2,6 +2,7 @@ package lnd
import (
"encoding/hex"
"errors"
"fmt"
"io/ioutil"
"net"
@@ -161,7 +162,7 @@ type chainControl struct {
// full-node, another backed by a running bitcoind full-node, and the other
// backed by a running neutrino light client instance. When running with a
// neutrino light client instance, `neutrinoCS` must be non-nil.
func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
func newChainControlFromConfig(cfg *Config, localDB, remoteDB *channeldb.DB,
privateWalletPw, publicWalletPw []byte, birthday time.Time,
recoveryWindow uint32, wallet *wallet.Wallet,
neutrinoCS *neutrino.ChainService) (*chainControl, error) {
@@ -212,8 +213,8 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
Birthday: birthday,
RecoveryWindow: recoveryWindow,
DataDir: homeChainConfig.ChainDir,
NetParams: activeNetParams.Params,
CoinType: activeNetParams.CoinType,
NetParams: cfg.ActiveNetParams.Params,
CoinType: cfg.ActiveNetParams.CoinType,
Wallet: wallet,
}
@@ -225,8 +226,9 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
if cfg.HeightHintCacheQueryDisable {
ltndLog.Infof("Height Hint Cache Queries disabled")
}
// Initialize the height hint cache within the chain directory.
hintCache, err := chainntnfs.NewHeightHintCache(heightHintCacheConfig, chanDB)
hintCache, err := chainntnfs.NewHeightHintCache(heightHintCacheConfig, localDB)
if err != nil {
return nil, fmt.Errorf("unable to initialize height hint "+
"cache: %v", err)
@@ -248,25 +250,19 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
return nil, err
}
// If the user provided an API for fee estimation, activate it now.
// Map the deprecated neutrino feeurl flag to the general fee
// url.
if cfg.NeutrinoMode.FeeURL != "" {
ltndLog.Infof("Using API fee estimator!")
estimator := chainfee.NewWebAPIEstimator(
chainfee.SparseConfFeeSource{
URL: cfg.NeutrinoMode.FeeURL,
},
defaultBitcoinStaticFeePerKW,
)
if err := estimator.Start(); err != nil {
return nil, err
if cfg.FeeURL != "" {
return nil, errors.New("feeurl and " +
"neutrino.feeurl are mutually exclusive")
}
cc.feeEstimator = estimator
cfg.FeeURL = cfg.NeutrinoMode.FeeURL
}
walletConfig.ChainSource = chain.NewNeutrinoClient(
activeNetParams.Params, neutrinoCS,
cfg.ActiveNetParams.Params, neutrinoCS,
)
case "bitcoind", "litecoind":
@@ -290,7 +286,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
// btcd, which picks a different port so that btcwallet
// can use the same RPC port as bitcoind. We convert
// this back to the btcwallet/bitcoind port.
rpcPort, err := strconv.Atoi(activeNetParams.rpcPort)
rpcPort, err := strconv.Atoi(cfg.ActiveNetParams.rpcPort)
if err != nil {
return nil, err
}
@@ -318,7 +314,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
// Establish the connection to bitcoind and create the clients
// required for our relevant subsystems.
bitcoindConn, err := chain.NewBitcoindConn(
activeNetParams.Params, bitcoindHost,
cfg.ActiveNetParams.Params, bitcoindHost,
bitcoindMode.RPCUser, bitcoindMode.RPCPass,
bitcoindMode.ZMQPubRawBlock, bitcoindMode.ZMQPubRawTx,
5*time.Second,
@@ -333,7 +329,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
}
cc.chainNotifier = bitcoindnotify.New(
bitcoindConn, activeNetParams.Params, hintCache, hintCache,
bitcoindConn, cfg.ActiveNetParams.Params, hintCache, hintCache,
)
cc.chainView = chainview.NewBitcoindFilteredChainView(bitcoindConn)
walletConfig.ChainSource = bitcoindConn.NewBitcoindClient()
@@ -365,9 +361,6 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
if err != nil {
return nil, err
}
if err := cc.feeEstimator.Start(); err != nil {
return nil, err
}
} else if cfg.Litecoin.Active && !cfg.Litecoin.RegTest {
ltndLog.Infof("Initializing litecoind backed fee estimator in "+
"%s mode", bitcoindMode.EstimateMode)
@@ -384,9 +377,6 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
if err != nil {
return nil, err
}
if err := cc.feeEstimator.Start(); err != nil {
return nil, err
}
}
case "btcd", "ltcd":
// Otherwise, we'll be speaking directly via RPC to a node.
@@ -431,7 +421,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
btcdHost = btcdMode.RPCHost
} else {
btcdHost = fmt.Sprintf("%v:%v", btcdMode.RPCHost,
activeNetParams.rpcPort)
cfg.ActiveNetParams.rpcPort)
}
btcdUser := btcdMode.RPCUser
@@ -447,7 +437,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
DisableAutoReconnect: false,
}
cc.chainNotifier, err = btcdnotify.New(
rpcConfig, activeNetParams.Params, hintCache, hintCache,
rpcConfig, cfg.ActiveNetParams.Params, hintCache, hintCache,
)
if err != nil {
return nil, err
@@ -463,7 +453,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
// Create a special websockets rpc client for btcd which will be used
// by the wallet for notifications, calls, etc.
chainRPC, err := chain.NewRPCClient(activeNetParams.Params, btcdHost,
chainRPC, err := chain.NewRPCClient(cfg.ActiveNetParams.Params, btcdHost,
btcdUser, btcdPass, rpcCert, false, 20)
if err != nil {
return nil, err
@@ -489,15 +479,34 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
if err != nil {
return nil, err
}
if err := cc.feeEstimator.Start(); err != nil {
return nil, err
}
}
default:
return nil, fmt.Errorf("unknown node type: %s",
homeChainConfig.Node)
}
// Override default fee estimator if an external service is specified.
if cfg.FeeURL != "" {
// Do not cache fees on regtest to make it easier to execute
// manual or automated test cases.
cacheFees := !cfg.Bitcoin.RegTest
ltndLog.Infof("Using external fee estimator %v: cached=%v",
cfg.FeeURL, cacheFees)
cc.feeEstimator = chainfee.NewWebAPIEstimator(
chainfee.SparseConfFeeSource{
URL: cfg.FeeURL,
},
!cacheFees,
)
}
// Start fee estimator.
if err := cc.feeEstimator.Start(); err != nil {
return nil, err
}
wc, err := btcwallet.New(*walletConfig)
if err != nil {
fmt.Printf("unable to create wallet controller: %v\n", err)
@@ -516,14 +525,14 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
}
keyRing := keychain.NewBtcWalletKeyRing(
wc.InternalWallet(), activeNetParams.CoinType,
wc.InternalWallet(), cfg.ActiveNetParams.CoinType,
)
cc.keyRing = keyRing
// Create, and start the lnwallet, which handles the core payment
// channel logic, and exposes control via proxy state machines.
walletCfg := lnwallet.Config{
Database: chanDB,
Database: remoteDB,
Notifier: cc.chainNotifier,
WalletController: wc,
Signer: cc.signer,
@@ -531,7 +540,7 @@ func newChainControlFromConfig(cfg *Config, chanDB *channeldb.DB,
SecretKeyRing: keyRing,
ChainIO: cc.chainIO,
DefaultConstraints: channelConstraints,
NetParams: *activeNetParams.Params,
NetParams: *cfg.ActiveNetParams.Params,
}
lnWallet, err := lnwallet.NewLightningWallet(walletCfg)
if err != nil {
@@ -733,7 +742,7 @@ func initNeutrinoBackend(cfg *Config, chainDir string) (*neutrino.ChainService,
// match the behavior of btcwallet.
dbPath := filepath.Join(
chainDir,
normalizeNetwork(activeNetParams.Name),
normalizeNetwork(cfg.ActiveNetParams.Name),
)
// Ensure that the neutrino db path exists.
@@ -762,11 +771,14 @@ func initNeutrinoBackend(cfg *Config, chainDir string) (*neutrino.ChainService,
config := neutrino.Config{
DataDir: dbPath,
Database: db,
ChainParams: *activeNetParams.Params,
ChainParams: *cfg.ActiveNetParams.Params,
AddPeers: cfg.NeutrinoMode.AddPeers,
ConnectPeers: cfg.NeutrinoMode.ConnectPeers,
Dialer: func(addr net.Addr) (net.Conn, error) {
return cfg.net.Dial(addr.Network(), addr.String())
return cfg.net.Dial(
addr.Network(), addr.String(),
cfg.ConnectionTimeout,
)
},
NameResolver: func(host string) ([]net.IP, error) {
addrs, err := cfg.net.LookupHost(host)
+297 -97
View File
@@ -5,7 +5,7 @@ import (
"time"
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/routing/route"
"github.com/lightningnetwork/lnd/clock"
)
type eventType int
@@ -28,74 +28,244 @@ func (e eventType) String() string {
return "unknown"
}
// channelEvent is a a timestamped event which is observed on a per channel
// basis.
type channelEvent struct {
type event struct {
timestamp time.Time
eventType eventType
}
// chanEventLog stores all events that have occurred over a channel's lifetime.
type chanEventLog struct {
// channelPoint is the outpoint for the channel's funding transaction.
channelPoint wire.OutPoint
// peerLog tracks events for a peer and its channels. If we currently have no
// channels with the peer, it will simply track its current online state. If we
// do have channels open with the peer, it will track the peer's online and
// offline events so that we can calculate uptime for our channels. A single
// event log is used for these online and offline events, and uptime for a
// channel is calculated by examining a subsection of this log.
type peerLog struct {
// online stores whether the peer is currently online.
online bool
// peer is the compressed public key of the peer being monitored.
peer route.Vertex
// onlineEvents is a log of timestamped events observed for the peer
// that we have committed to allocating memory to.
onlineEvents []*event
// events is a log of timestamped events observed for the channel.
events []*channelEvent
// stagedEvent represents an event that is pending addition to the
// events list. It has not yet been added because we rate limit the
// frequency that we store events at. We need to store this value
// in the log (rather than just ignore events) so that we can flush the
// aggregate outcome to our event log once the rate limiting period has
// ended.
//
// Take the following example:
// - Peer online event recorded
// - Peer offline event, not recorded due to rate limit
// - No more events, we incorrectly believe our peer to be online
// Instead of skipping events, we stage the most recent event during the
// rate limited period so that we know what happened (on aggregate)
// while we were rate limiting events.
//
// Note that we currently only store offline/online events so we can
// use this field to track our online state. With the addition of other
// event types, we need to only stage online/offline events, or split
// them out.
stagedEvent *event
// now is expected to return the current time. It is supplied as an
// external function to enable deterministic unit tests.
now func() time.Time
// flapCount is the number of times this peer has been observed as
// going offline.
flapCount int
// openedAt tracks the first time this channel was seen. This is not
// necessarily the time that it confirmed on chain because channel events
// are not persisted at present.
openedAt time.Time
// lastFlap is the timestamp of the last flap we recorded for the peer.
// This value will be nil if we have never recorded a flap for the peer.
lastFlap *time.Time
// closedAt is the time that the channel was closed. If the channel has not
// been closed yet, it is zero.
closedAt time.Time
// clock allows creation of deterministic unit tests.
clock clock.Clock
// channels contains a set of currently open channels. Channels will be
// added and removed from this map as they are opened and closed.
channels map[wire.OutPoint]*channelInfo
}
// newEventLog creates an event log for a channel with the openedAt time set.
func newEventLog(channelPoint wire.OutPoint, peer route.Vertex,
now func() time.Time) *chanEventLog {
// newPeerLog creates a log for a peer, taking its historical flap count and
// last flap time as parameters. These values may be zero/nil if we have no
// record of historical flap count for the peer.
func newPeerLog(clock clock.Clock, flapCount int,
lastFlap *time.Time) *peerLog {
eventlog := &chanEventLog{
channelPoint: channelPoint,
peer: peer,
now: now,
openedAt: now(),
return &peerLog{
clock: clock,
flapCount: flapCount,
lastFlap: lastFlap,
channels: make(map[wire.OutPoint]*channelInfo),
}
}
// channelInfo contains information about a channel.
type channelInfo struct {
// openedAt tracks the first time this channel was seen. This is not
// necessarily the time that it confirmed on chain because channel
// events are not persisted at present.
openedAt time.Time
}
func newChannelInfo(openedAt time.Time) *channelInfo {
return &channelInfo{
openedAt: openedAt,
}
}
// onlineEvent records a peer online or offline event in the log and increments
// the peer's flap count.
func (p *peerLog) onlineEvent(online bool) {
eventTime := p.clock.Now()
// If we have a non-nil last flap time, potentially apply a cooldown
// factor to the peer's flap count before we rate limit it. This allows
// us to decrease the penalty for historical flaps over time, provided
// the peer has not flapped for a while.
if p.lastFlap != nil {
p.flapCount = cooldownFlapCount(
p.clock.Now(), p.flapCount, *p.lastFlap,
)
}
return eventlog
}
// Record flap count information and online state regardless of whether
// we have any channels open with this peer.
p.flapCount++
p.lastFlap = &eventTime
p.online = online
// close sets the closing time for an event log.
func (e *chanEventLog) close() {
e.closedAt = e.now()
}
// add appends an event with the given type and current time to the event log.
// The open time for the eventLog will be set to the event's timestamp if it is
// not set yet.
func (e *chanEventLog) add(eventType eventType) {
// If the channel is already closed, return early without adding an event.
if !e.closedAt.IsZero() {
// If we have no channels currently open with the peer, we do not want
// to commit resources to tracking their online state beyond a simple
// online boolean, so we exit early.
if p.channelCount() == 0 {
return
}
// Add the event to the eventLog with the current timestamp.
event := &channelEvent{
timestamp: e.now(),
p.addEvent(online, eventTime)
}
// addEvent records an online or offline event in our event log. and increments
// the peer's flap count.
func (p *peerLog) addEvent(online bool, time time.Time) {
eventType := peerOnlineEvent
if !online {
eventType = peerOfflineEvent
}
event := &event{
timestamp: time,
eventType: eventType,
}
e.events = append(e.events, event)
log.Debugf("Channel %v recording event: %v", e.channelPoint, eventType)
// If we have no staged events, we can just stage this event and return.
if p.stagedEvent == nil {
p.stagedEvent = event
return
}
// We get the amount of time we require between events according to
// peer flap count.
aggregation := getRateLimit(p.flapCount)
nextRecordTime := p.stagedEvent.timestamp.Add(aggregation)
flushEvent := nextRecordTime.Before(event.timestamp)
// If enough time has passed since our last staged event, we add our
// event to our in-memory list.
if flushEvent {
p.onlineEvents = append(p.onlineEvents, p.stagedEvent)
}
// Finally, we replace our staged event with the new event we received.
p.stagedEvent = event
}
// addChannel adds a channel to our log. If we have not tracked any online
// events for our peer yet, we create one with our peer's current online state
// so that we know the state that the peer had at channel start, which is
// required to calculate uptime over the channel's lifetime.
func (p *peerLog) addChannel(channelPoint wire.OutPoint) error {
_, ok := p.channels[channelPoint]
if ok {
return fmt.Errorf("channel: %v already present", channelPoint)
}
openTime := p.clock.Now()
p.channels[channelPoint] = newChannelInfo(openTime)
// If we do not have any online events tracked for our peer (which is
// the case when we have no other channels open with the peer), we add
// an event with the peer's current online state so that we know that
// starting state for this peer when a channel was connected (which
// allows us to calculate uptime over the lifetime of the channel).
if len(p.onlineEvents) == 0 {
p.addEvent(p.online, openTime)
}
return nil
}
// removeChannel removes a channel from our log. If we have no more channels
// with the peer after removing this one, we clear our list of events.
func (p *peerLog) removeChannel(channelPoint wire.OutPoint) error {
_, ok := p.channels[channelPoint]
if !ok {
return fmt.Errorf("channel: %v not present", channelPoint)
}
delete(p.channels, channelPoint)
// If we have no more channels in our event log, we can discard all of
// our online events in memory, since we don't need them anymore.
// TODO(carla): this could be done on a per channel basis.
if p.channelCount() == 0 {
p.onlineEvents = nil
p.stagedEvent = nil
}
return nil
}
// channelCount returns the number of channels that we currently have
// with the peer.
func (p *peerLog) channelCount() int {
return len(p.channels)
}
// channelUptime looks up a channel and returns the amount of time that the
// channel has been monitored for and its uptime over this period.
func (p *peerLog) channelUptime(channelPoint wire.OutPoint) (time.Duration,
time.Duration, error) {
channel, ok := p.channels[channelPoint]
if !ok {
return 0, 0, ErrChannelNotFound
}
now := p.clock.Now()
uptime, err := p.uptime(channel.openedAt, now)
if err != nil {
return 0, 0, err
}
return now.Sub(channel.openedAt), uptime, nil
}
// getFlapCount returns the peer's flap count and the timestamp that we last
// recorded a flap.
func (p *peerLog) getFlapCount() (int, *time.Time) {
return p.flapCount, p.lastFlap
}
// listEvents returns all of the events that our event log has tracked,
// including events that are staged for addition to our set of events but have
// not yet been committed to (because we rate limit and store only the aggregate
// outcome over a period).
func (p *peerLog) listEvents() []*event {
if p.stagedEvent == nil {
return p.onlineEvents
}
return append(p.onlineEvents, p.stagedEvent)
}
// onlinePeriod represents a period of time over which a peer was online.
@@ -106,17 +276,25 @@ type onlinePeriod struct {
// getOnlinePeriods returns a list of all the periods that the event log has
// recorded the remote peer as being online. In the unexpected case where there
// are no events, the function returns early. Online periods are defined as a
// peer online event which is terminated by a peer offline event. This function
// expects the event log provided to be ordered by ascending timestamp.
func (e *chanEventLog) getOnlinePeriods() []*onlinePeriod {
// peer online event which is terminated by a peer offline event. If the event
// log ends on a peer online event, it appends a final period which is
// calculated until the present. This function expects the event log provided
// to be ordered by ascending timestamp, and can tolerate multiple consecutive
// online or offline events.
func (p *peerLog) getOnlinePeriods() []*onlinePeriod {
events := p.listEvents()
// Return early if there are no events, there are no online periods.
if len(e.events) == 0 {
if len(events) == 0 {
return nil
}
var (
lastOnline time.Time
offline bool
// lastEvent tracks the last event that we had that was of
// a different type to our own. It is used to determine the
// start time of our online periods when we experience an
// offline event, and to track our last recorded state.
lastEvent *event
onlinePeriods []*onlinePeriod
)
@@ -126,57 +304,77 @@ func (e *chanEventLog) getOnlinePeriods() []*onlinePeriod {
// the online event and the present is not tracked. The type of the most
// recent event is tracked using the offline bool so that we can add a
// final online period if necessary.
for _, event := range e.events {
for _, event := range events {
switch event.eventType {
case peerOnlineEvent:
lastOnline = event.timestamp
offline = false
case peerOfflineEvent:
offline = true
// Do not add to uptime if there is no previous online timestamp,
// the event log has started with an offline event
if lastOnline.IsZero() {
continue
// If our previous event is nil, we just set it and
// break out of the switch.
if lastEvent == nil {
lastEvent = event
break
}
// The eventLog has recorded an offline event, having previously
// been online so we add an online period to to set of online periods.
onlinePeriods = append(onlinePeriods, &onlinePeriod{
start: lastOnline,
end: event.timestamp,
})
// If our previous event was an offline event, we update
// it to this event. We do not do this if it was an
// online event because duplicate online events would
// progress our online timestamp forward (rather than
// keep it at our earliest online event timestamp).
if lastEvent.eventType == peerOfflineEvent {
lastEvent = event
}
case peerOfflineEvent:
// If our previous event is nil, we just set it and
// break out of the switch since we cannot record an
// online period from this single event.
if lastEvent == nil {
lastEvent = event
break
}
// If the last event we saw was an online event, we
// add an online period to our set and progress our
// previous event to this offline event. We do not
// do this if we have had duplicate offline events
// because we would be tracking the most recent offline
// event (rather than keep it at our earliest offline
// event timestamp).
if lastEvent.eventType == peerOnlineEvent {
onlinePeriods = append(
onlinePeriods, &onlinePeriod{
start: lastEvent.timestamp,
end: event.timestamp,
},
)
lastEvent = event
}
}
}
// If the last event was an peer offline event, we do not need to calculate
// a final online period and can return online periods as is.
if offline {
// If the last event was an peer offline event, we do not need to
// calculate a final online period and can return online periods as is.
if lastEvent.eventType == peerOfflineEvent {
return onlinePeriods
}
// The log ended on an online event, so we need to add a final online event.
// If the channel is closed, this period is until channel closure. It it is
// still open, we calculate it until the present.
endTime := e.closedAt
if endTime.IsZero() {
endTime = e.now()
// The log ended on an online event, so we need to add a final online
// period which terminates at the present.
finalEvent := &onlinePeriod{
start: lastEvent.timestamp,
end: p.clock.Now(),
}
// Add the final online period to the set and return.
return append(onlinePeriods, &onlinePeriod{
start: lastOnline,
end: endTime,
})
return append(onlinePeriods, finalEvent)
}
// uptime calculates the total uptime we have recorded for a channel over the
// uptime calculates the total uptime we have recorded for a peer over the
// inclusive range specified. An error is returned if the end of the range is
// before the start or a zero end time is returned.
func (e *chanEventLog) uptime(start, end time.Time) (time.Duration, error) {
// Error if we are provided with an invalid range to calculate uptime for.
func (p *peerLog) uptime(start, end time.Time) (time.Duration, error) {
// Error if we are provided with an invalid range to calculate uptime
// for.
if end.Before(start) {
return 0, fmt.Errorf("end time: %v before start time: %v",
end, start)
@@ -187,26 +385,28 @@ func (e *chanEventLog) uptime(start, end time.Time) (time.Duration, error) {
var uptime time.Duration
for _, p := range e.getOnlinePeriods() {
// The online period ends before the range we're looking at, so we can
// skip over it.
for _, p := range p.getOnlinePeriods() {
// The online period ends before the range we're looking at, so
// we can skip over it.
if p.end.Before(start) {
continue
}
// The online period starts after the range we're looking at, so can
// stop calculating uptime.
// The online period starts after the range we're looking at, so
// can stop calculating uptime.
if p.start.After(end) {
break
}
// If the online period starts before our range, shift the start time up
// so that we only calculate uptime from the start of our range.
// If the online period starts before our range, shift the start
// time up so that we only calculate uptime from the start of
// our range.
if p.start.Before(start) {
p.start = start
}
// If the online period ends before our range, shift the end time
// forward so that we only calculate uptime until the end of the range.
// If the online period ends before our range, shift the end
// time forward so that we only calculate uptime until the end
// of the range.
if p.end.After(end) {
p.end = end
}
+380 -211
View File
@@ -3,77 +3,269 @@ package chanfitness
import (
"testing"
"time"
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/clock"
"github.com/stretchr/testify/require"
)
// TestAdd tests adding events to an event log. It tests the case where the
// channel is open, and should have an event added, and the case where it is
// closed and the event should not be added.
func TestAdd(t *testing.T) {
tests := []struct {
name string
eventLog *chanEventLog
event eventType
expected []eventType
}{
{
name: "Channel open",
eventLog: &chanEventLog{
now: time.Now,
},
event: peerOnlineEvent,
expected: []eventType{peerOnlineEvent},
},
{
name: "Channel closed, event not added",
eventLog: &chanEventLog{
now: time.Now,
},
event: peerOnlineEvent,
expected: []eventType{},
},
// TestPeerLog tests the functionality of the peer log struct.
func TestPeerLog(t *testing.T) {
clock := clock.NewTestClock(testNow)
peerLog := newPeerLog(clock, 0, nil)
// assertFlapCount is a helper that asserts that our peer's flap count
// and timestamp is set to expected values.
assertFlapCount := func(expectedCount int, expectedTs *time.Time) {
flapCount, flapTs := peerLog.getFlapCount()
require.Equal(t, expectedCount, flapCount)
require.Equal(t, expectedTs, flapTs)
}
for _, test := range tests {
test := test
require.Zero(t, peerLog.channelCount())
require.False(t, peerLog.online)
assertFlapCount(0, nil)
t.Run(test.name, func(t *testing.T) {
test.eventLog.add(test.event)
// Test that looking up an unknown channel fails.
_, _, err := peerLog.channelUptime(wire.OutPoint{Index: 1})
require.Error(t, err)
for i, e := range test.expected {
if test.eventLog.events[i].eventType != e {
t.Fatalf("Expected event type: %v, got: %v",
e, test.eventLog.events[i].eventType)
}
}
})
lastFlap := clock.Now()
// Add an offline event, since we have no channels, we do not expect
// to have any online periods recorded for our peer. However, we should
// increment our flap count for the peer.
peerLog.onlineEvent(false)
require.Len(t, peerLog.getOnlinePeriods(), 0)
assertFlapCount(1, &lastFlap)
// Bump our test clock's time by an hour so that we can create an online
// event with a distinct time.
lastFlap = testNow.Add(time.Hour)
clock.SetTime(lastFlap)
// Likewise, if we have an online event, nothing beyond the online state
// of our peer log should change, but our flap count should change.
peerLog.onlineEvent(true)
require.Len(t, peerLog.getOnlinePeriods(), 0)
assertFlapCount(2, &lastFlap)
// Add a channel and assert that we have one channel listed. Since this
// is the first channel we track for the peer, we expect an online
// event to be added, however, our flap count should not change because
// this is not a new online event, we are just copying one into our log
// for our purposes.
chan1 := wire.OutPoint{
Index: 1,
}
require.NoError(t, peerLog.addChannel(chan1))
require.Equal(t, 1, peerLog.channelCount())
assertFlapCount(2, &lastFlap)
// Assert that we can now successfully get our added channel.
_, _, err = peerLog.channelUptime(chan1)
require.NoError(t, err)
// Bump our test clock's time so that our current time is different to
// channel open time.
lastFlap = clock.Now().Add(time.Hour)
clock.SetTime(lastFlap)
// Now that we have added a channel and an hour has passed, we expect
// our uptime and lifetime to both equal an hour.
lifetime, uptime, err := peerLog.channelUptime(chan1)
require.NoError(t, err)
require.Equal(t, time.Hour, lifetime)
require.Equal(t, time.Hour, uptime)
// Add an offline event for our peer and assert that our flap count is
// incremented.
peerLog.onlineEvent(false)
assertFlapCount(3, &lastFlap)
// Now we add another channel to our store and assert that we now report
// two channels for this peer.
chan2 := wire.OutPoint{
Index: 2,
}
require.NoError(t, peerLog.addChannel(chan2))
require.Equal(t, 2, peerLog.channelCount())
// Progress our time again, so that our peer has now been offline for
// two hours.
now := lastFlap.Add(time.Hour * 2)
clock.SetTime(now)
// Our first channel should report as having been monitored for three
// hours, but only online for one of those hours.
lifetime, uptime, err = peerLog.channelUptime(chan1)
require.NoError(t, err)
require.Equal(t, time.Hour*3, lifetime)
require.Equal(t, time.Hour, uptime)
// Remove our first channel and check that we can still correctly query
// uptime for the second channel.
require.NoError(t, peerLog.removeChannel(chan1))
require.Equal(t, 1, peerLog.channelCount())
// Our second channel, which was created when our peer was offline,
// should report as having been monitored for two hours, but have zero
// uptime.
lifetime, uptime, err = peerLog.channelUptime(chan2)
require.NoError(t, err)
require.Equal(t, time.Hour*2, lifetime)
require.Equal(t, time.Duration(0), uptime)
// Finally, remove our second channel and assert that our peer cleans
// up its in memory set of events but keeps its flap count record.
require.NoError(t, peerLog.removeChannel(chan2))
require.Equal(t, 0, peerLog.channelCount())
require.Len(t, peerLog.onlineEvents, 0)
assertFlapCount(3, &lastFlap)
require.Len(t, peerLog.listEvents(), 0)
require.Nil(t, peerLog.stagedEvent)
}
// TestRateLimitAdd tests the addition of events to the event log with rate
// limiting in place.
func TestRateLimitAdd(t *testing.T) {
// Create a mock clock specifically for this test so that we can
// progress time without affecting the other tests.
mockedClock := clock.NewTestClock(testNow)
// Create a new peer log.
peerLog := newPeerLog(mockedClock, 0, nil)
require.Nil(t, peerLog.stagedEvent)
// Create a channel for our peer log, otherwise it will not track online
// events.
require.NoError(t, peerLog.addChannel(wire.OutPoint{}))
// First, we add an event to the event log. Since we have no previous
// events, we expect this event to staged immediately.
peerEvent := &event{
timestamp: testNow,
eventType: peerOfflineEvent,
}
peerLog.onlineEvent(false)
require.Equal(t, peerEvent, peerLog.stagedEvent)
// We immediately add another event to our event log. We expect our
// staged event to be replaced with this new event, because insufficient
// time has passed since our last event.
peerEvent = &event{
timestamp: testNow,
eventType: peerOnlineEvent,
}
peerLog.onlineEvent(true)
require.Equal(t, peerEvent, peerLog.stagedEvent)
// We get the amount of time that we need to pass before we record an
// event from our rate limiting tiers. We then progress our test clock
// to just after this point.
delta := getRateLimit(peerLog.flapCount)
newNow := testNow.Add(delta + 1)
mockedClock.SetTime(newNow)
// Now, when we add an event, we expect our staged event to be added
// to our events list and for our new event to be staged.
newEvent := &event{
timestamp: newNow,
eventType: peerOfflineEvent,
}
peerLog.onlineEvent(false)
require.Equal(t, []*event{peerEvent}, peerLog.onlineEvents)
require.Equal(t, newEvent, peerLog.stagedEvent)
// Now, we test the case where we add many events to our log. We expect
// our set of events to be untouched, but for our staged event to be
// updated.
nextEvent := &event{
timestamp: newNow,
eventType: peerOnlineEvent,
}
for i := 0; i < 5; i++ {
// We flip the kind of event for each type so that we can check
// that our staged event is definitely changing each time.
if i%2 == 0 {
nextEvent.eventType = peerOfflineEvent
} else {
nextEvent.eventType = peerOnlineEvent
}
online := nextEvent.eventType == peerOnlineEvent
peerLog.onlineEvent(online)
require.Equal(t, []*event{peerEvent}, peerLog.onlineEvents)
require.Equal(t, nextEvent, peerLog.stagedEvent)
}
// Now, we test the case where a peer's flap count is cooled down
// because it has not flapped for a while. Set our peer's flap count so
// that we fall within our second rate limiting tier and assert that we
// are at this level.
peerLog.flapCount = rateLimitScale + 1
rateLimit := getRateLimit(peerLog.flapCount)
require.Equal(t, rateLimits[1], rateLimit)
// Progress our clock to the point where we will have our flap count
// cooled.
newNow = mockedClock.Now().Add(flapCountCooldownPeriod)
mockedClock.SetTime(newNow)
// Add an online event, and expect it to be staged.
onlineEvent := &event{
timestamp: newNow,
eventType: peerOnlineEvent,
}
peerLog.onlineEvent(true)
require.Equal(t, onlineEvent, peerLog.stagedEvent)
// Progress our clock by the rate limit level that we will be on if
// our flap rate is cooled down to a lower level.
newNow = mockedClock.Now().Add(rateLimits[0] + 1)
mockedClock.SetTime(newNow)
// Add another event. We expect this event to be staged and our previous
// event to be flushed to the event log (because our cooldown has been
// applied).
offlineEvent := &event{
timestamp: newNow,
eventType: peerOfflineEvent,
}
peerLog.onlineEvent(false)
require.Equal(t, offlineEvent, peerLog.stagedEvent)
flushedEventIdx := len(peerLog.onlineEvents) - 1
require.Equal(
t, onlineEvent, peerLog.onlineEvents[flushedEventIdx],
)
}
// TestGetOnlinePeriod tests the getOnlinePeriod function. It tests the case
// where no events present, and the case where an additional online period
// must be added because the event log ends on an online event.
func TestGetOnlinePeriod(t *testing.T) {
// Set time for consistent testing.
now := time.Now()
fourHoursAgo := now.Add(time.Hour * -4)
threeHoursAgo := now.Add(time.Hour * -3)
twoHoursAgo := now.Add(time.Hour * -2)
oneHourAgo := now.Add(time.Hour * -1)
fourHoursAgo := testNow.Add(time.Hour * -4)
threeHoursAgo := testNow.Add(time.Hour * -3)
twoHoursAgo := testNow.Add(time.Hour * -2)
tests := []struct {
name string
events []*channelEvent
events []*event
expectedOnline []*onlinePeriod
openedAt time.Time
closedAt time.Time
}{
{
name: "No events",
name: "no events",
},
{
name: "Start on online period",
events: []*channelEvent{
name: "start on online period",
events: []*event{
{
timestamp: threeHoursAgo,
eventType: peerOnlineEvent,
@@ -91,8 +283,8 @@ func TestGetOnlinePeriod(t *testing.T) {
},
},
{
name: "Start on offline period",
events: []*channelEvent{
name: "start on offline period",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOfflineEvent,
@@ -100,8 +292,8 @@ func TestGetOnlinePeriod(t *testing.T) {
},
},
{
name: "End on an online period, channel not closed",
events: []*channelEvent{
name: "end on an online period",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
@@ -110,25 +302,88 @@ func TestGetOnlinePeriod(t *testing.T) {
expectedOnline: []*onlinePeriod{
{
start: fourHoursAgo,
end: now,
end: testNow,
},
},
},
{
name: "End on an online period, channel closed",
events: []*channelEvent{
name: "duplicate online events",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
{
timestamp: threeHoursAgo,
eventType: peerOnlineEvent,
},
},
expectedOnline: []*onlinePeriod{
{
start: fourHoursAgo,
end: oneHourAgo,
end: testNow,
},
},
},
{
name: "duplicate offline events",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOfflineEvent,
},
{
timestamp: threeHoursAgo,
eventType: peerOfflineEvent,
},
},
expectedOnline: nil,
},
{
name: "duplicate online then offline",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
{
timestamp: threeHoursAgo,
eventType: peerOnlineEvent,
},
{
timestamp: twoHoursAgo,
eventType: peerOfflineEvent,
},
},
expectedOnline: []*onlinePeriod{
{
start: fourHoursAgo,
end: twoHoursAgo,
},
},
},
{
name: "duplicate offline then online",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOfflineEvent,
},
{
timestamp: threeHoursAgo,
eventType: peerOfflineEvent,
},
{
timestamp: twoHoursAgo,
eventType: peerOnlineEvent,
},
},
expectedOnline: []*onlinePeriod{
{
start: twoHoursAgo,
end: testNow,
},
},
closedAt: oneHourAgo,
},
}
@@ -136,33 +391,16 @@ func TestGetOnlinePeriod(t *testing.T) {
test := test
t.Run(test.name, func(t *testing.T) {
score := &chanEventLog{
events: test.events,
now: func() time.Time {
return now
},
openedAt: test.openedAt,
closedAt: test.closedAt,
t.Parallel()
score := &peerLog{
onlineEvents: test.events,
clock: clock.NewTestClock(testNow),
}
online := score.getOnlinePeriods()
if len(online) != len(test.expectedOnline) {
t.Fatalf("Expectd: %v online periods, got: %v",
len(test.expectedOnline), len(online))
}
for i, o := range test.expectedOnline {
if online[i].start != o.start {
t.Errorf("Expected start: %v, got %v", o.start,
online[i].start)
}
if online[i].end != o.end {
t.Errorf("Expected end: %v, got %v", o.end,
online[i].end)
}
}
require.Equal(t, test.expectedOnline, online)
})
}
@@ -170,48 +408,39 @@ func TestGetOnlinePeriod(t *testing.T) {
// TestUptime tests channel uptime calculation based on its event log.
func TestUptime(t *testing.T) {
// Set time for consistent testing.
now := time.Now()
fourHoursAgo := now.Add(time.Hour * -4)
threeHoursAgo := now.Add(time.Hour * -3)
twoHoursAgo := now.Add(time.Hour * -2)
oneHourAgo := now.Add(time.Hour * -1)
fourHoursAgo := testNow.Add(time.Hour * -4)
threeHoursAgo := testNow.Add(time.Hour * -3)
twoHoursAgo := testNow.Add(time.Hour * -2)
oneHourAgo := testNow.Add(time.Hour * -1)
tests := []struct {
name string
// opened at is the time the channel was recorded as being open, and is
// never expected to be zero.
openedAt time.Time
// events is the set of event log that we are calculating uptime
// for.
events []*event
// closed at is the tim the channel was recorded as being closed, and
// can have a zero value if the.
closedAt time.Time
// events is the set of event log that we are calculating uptime for.
events []*channelEvent
// startTime is the beginning of the period that we are calculating
// uptime for, it cannot have a zero value.
// startTime is the beginning of the period that we are
// calculating uptime for, it cannot have a zero value.
startTime time.Time
// endTime is the end of the period that we are calculating uptime for,
// it cannot have a zero value.
// endTime is the end of the period that we are calculating
// uptime for, it cannot have a zero value.
endTime time.Time
// expectedUptime is the amount of uptime we expect to be calculated
// over the period specified by startTime and endTime.
// expectedUptime is the amount of uptime we expect to be
// calculated over the period specified by startTime and
// endTime.
expectedUptime time.Duration
// expectErr is set to true if we expect an error to be returned when
// calling the uptime function
// expectErr is set to true if we expect an error to be returned
// when calling the uptime function.
expectErr bool
}{
{
name: "End before start",
endTime: threeHoursAgo,
startTime: now,
startTime: testNow,
expectErr: true,
},
{
@@ -219,64 +448,20 @@ func TestUptime(t *testing.T) {
expectErr: true,
},
{
name: "Online event and closed",
openedAt: fourHoursAgo,
closedAt: oneHourAgo,
events: []*channelEvent{
name: "online event and no offline",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
},
startTime: fourHoursAgo,
endTime: now,
expectedUptime: time.Hour * 3,
},
{
name: "Online event and not closed",
openedAt: fourHoursAgo,
events: []*channelEvent{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
},
startTime: fourHoursAgo,
endTime: now,
endTime: testNow,
expectedUptime: time.Hour * 4,
},
{
name: "Offline event and closed",
openedAt: fourHoursAgo,
closedAt: threeHoursAgo,
events: []*channelEvent{
{
timestamp: fourHoursAgo,
eventType: peerOfflineEvent,
},
},
startTime: fourHoursAgo,
endTime: now,
},
{
name: "Online event before close",
openedAt: fourHoursAgo,
closedAt: oneHourAgo,
events: []*channelEvent{
{
timestamp: twoHoursAgo,
eventType: peerOnlineEvent,
},
},
startTime: fourHoursAgo,
endTime: now,
expectedUptime: time.Hour,
},
{
name: "Online then offline event",
openedAt: fourHoursAgo,
closedAt: oneHourAgo,
events: []*channelEvent{
name: "online then offline event",
events: []*event{
{
timestamp: threeHoursAgo,
eventType: peerOnlineEvent,
@@ -287,44 +472,40 @@ func TestUptime(t *testing.T) {
},
},
startTime: fourHoursAgo,
endTime: now,
endTime: testNow,
expectedUptime: time.Hour,
},
{
name: "Online event before uptime period",
openedAt: fourHoursAgo,
closedAt: oneHourAgo,
events: []*channelEvent{
name: "online event before uptime period",
events: []*event{
{
timestamp: threeHoursAgo,
eventType: peerOnlineEvent,
},
},
startTime: twoHoursAgo,
endTime: now,
expectedUptime: time.Hour,
},
{
name: "Offline event after uptime period",
openedAt: fourHoursAgo,
events: []*channelEvent{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
{
timestamp: now.Add(time.Hour),
eventType: peerOfflineEvent,
},
},
startTime: twoHoursAgo,
endTime: now,
endTime: testNow,
expectedUptime: time.Hour * 2,
},
{
name: "All events within period",
openedAt: fourHoursAgo,
events: []*channelEvent{
name: "offline event after uptime period",
events: []*event{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
{
timestamp: testNow.Add(time.Hour),
eventType: peerOfflineEvent,
},
},
startTime: twoHoursAgo,
endTime: testNow,
expectedUptime: time.Hour * 2,
},
{
name: "all events within period",
events: []*event{
{
timestamp: twoHoursAgo,
eventType: peerOnlineEvent,
@@ -335,31 +516,30 @@ func TestUptime(t *testing.T) {
expectedUptime: time.Hour,
},
{
name: "Multiple online and offline",
openedAt: now.Add(time.Hour * -8),
events: []*channelEvent{
name: "multiple online and offline",
events: []*event{
{
timestamp: now.Add(time.Hour * -7),
timestamp: testNow.Add(time.Hour * -7),
eventType: peerOnlineEvent,
},
{
timestamp: now.Add(time.Hour * -6),
timestamp: testNow.Add(time.Hour * -6),
eventType: peerOfflineEvent,
},
{
timestamp: now.Add(time.Hour * -5),
timestamp: testNow.Add(time.Hour * -5),
eventType: peerOnlineEvent,
},
{
timestamp: now.Add(time.Hour * -4),
timestamp: testNow.Add(time.Hour * -4),
eventType: peerOfflineEvent,
},
{
timestamp: now.Add(time.Hour * -3),
timestamp: testNow.Add(time.Hour * -3),
eventType: peerOnlineEvent,
},
},
startTime: now.Add(time.Hour * -8),
startTime: testNow.Add(time.Hour * -8),
endTime: oneHourAgo,
expectedUptime: time.Hour * 4,
},
@@ -369,27 +549,16 @@ func TestUptime(t *testing.T) {
test := test
t.Run(test.name, func(t *testing.T) {
score := &chanEventLog{
events: test.events,
now: func() time.Time {
return now
},
openedAt: test.openedAt,
closedAt: test.closedAt,
score := &peerLog{
onlineEvents: test.events,
clock: clock.NewTestClock(testNow),
}
uptime, err := score.uptime(test.startTime, test.endTime)
if test.expectErr && err == nil {
t.Fatal("Expected an error, got nil")
}
if !test.expectErr && err != nil {
t.Fatalf("Expcted no error, got: %v", err)
}
if uptime != test.expectedUptime {
t.Errorf("Expected uptime: %v, got: %v",
test.expectedUptime, uptime)
}
uptime, err := score.uptime(
test.startTime, test.endTime,
)
require.Equal(t, test.expectErr, err != nil)
require.Equal(t, test.expectedUptime, uptime)
})
}
}
+317 -155
View File
@@ -18,19 +18,31 @@ import (
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channelnotifier"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/peernotifier"
"github.com/lightningnetwork/lnd/routing/route"
"github.com/lightningnetwork/lnd/subscribe"
"github.com/lightningnetwork/lnd/ticker"
)
const (
// FlapCountFlushRate determines how often we write peer total flap
// count to disk.
FlapCountFlushRate = time.Hour
)
var (
// errShuttingDown is returned when the store cannot respond to a query because
// it has received the shutdown signal.
// errShuttingDown is returned when the store cannot respond to a query
// because it has received the shutdown signal.
errShuttingDown = errors.New("channel event store shutting down")
// ErrChannelNotFound is returned when a query is made for a channel that
// the event store does not have knowledge of.
// ErrChannelNotFound is returned when a query is made for a channel
// that the event store does not have knowledge of.
ErrChannelNotFound = errors.New("channel not found in event store")
// ErrPeerNotFound is returned when a query is made for a channel
// that has a peer that the event store is not currently tracking.
ErrPeerNotFound = errors.New("peer not found in event store")
)
// ChannelEventStore maintains a set of event logs for the node's channels to
@@ -38,18 +50,14 @@ var (
type ChannelEventStore struct {
cfg *Config
// channels maps channel points to event logs.
channels map[wire.OutPoint]*chanEventLog
// peers tracks all of our currently monitored peers and their channels.
peers map[route.Vertex]peerMonitor
// peers tracks the current online status of peers based on online/offline
// events.
peers map[route.Vertex]bool
// chanInfoRequests serves requests for information about our channel.
chanInfoRequests chan channelInfoRequest
// lifespanRequests serves requests for the lifespan of channels.
lifespanRequests chan lifespanRequest
// uptimeRequests serves requests for the uptime of channels.
uptimeRequests chan uptimeRequest
// peerRequests serves requests for information about a peer.
peerRequests chan peerRequest
quit chan struct{}
@@ -60,49 +68,58 @@ type ChannelEventStore struct {
// activity. All elements of the config must be non-nil for the event store to
// operate.
type Config struct {
// SubscribeChannelEvents provides a subscription client which provides a
// stream of channel events.
SubscribeChannelEvents func() (*subscribe.Client, error)
// SubscribeChannelEvents provides a subscription client which provides
// a stream of channel events.
SubscribeChannelEvents func() (subscribe.Subscription, error)
// SubscribePeerEvents provides a subscription client which provides a
// stream of peer online/offline events.
SubscribePeerEvents func() (*subscribe.Client, error)
SubscribePeerEvents func() (subscribe.Subscription, error)
// GetOpenChannels provides a list of existing open channels which is used
// to populate the ChannelEventStore with a set of channels on startup.
// GetOpenChannels provides a list of existing open channels which is
// used to populate the ChannelEventStore with a set of channels on
// startup.
GetOpenChannels func() ([]*channeldb.OpenChannel, error)
// Clock is the time source that the subsystem uses, provided here
// for ease of testing.
Clock clock.Clock
// WriteFlapCounts records the flap count for a set of peers on disk.
WriteFlapCount func(map[route.Vertex]*channeldb.FlapCount) error
// ReadFlapCount gets the flap count for a peer on disk.
ReadFlapCount func(route.Vertex) (*channeldb.FlapCount, error)
// FlapCountTicker is a ticker which controls how often we flush our
// peer's flap count to disk.
FlapCountTicker ticker.Ticker
}
// lifespanRequest contains the channel ID required to query the store for a
// channel's lifespan and a blocking response channel on which the result is
// sent.
type lifespanRequest struct {
// peerFlapCountMap is the map used to map peers to flap counts, declared here
// to allow shorter function signatures.
type peerFlapCountMap map[route.Vertex]*channeldb.FlapCount
type channelInfoRequest struct {
peer route.Vertex
channelPoint wire.OutPoint
responseChan chan lifespanResponse
responseChan chan channelInfoResponse
}
// lifespanResponse contains the response to a lifespanRequest and an error if
// one occurred.
type lifespanResponse struct {
start time.Time
end time.Time
err error
type channelInfoResponse struct {
info *ChannelInfo
err error
}
// uptimeRequest contains the parameters required to query the store for a
// channel's uptime and a blocking response channel on which the result is sent.
type uptimeRequest struct {
channelPoint wire.OutPoint
startTime time.Time
endTime time.Time
responseChan chan uptimeResponse
type peerRequest struct {
peer route.Vertex
responseChan chan peerResponse
}
// uptimeResponse contains the response to an uptimeRequest and an error if one
// occurred.
type uptimeResponse struct {
uptime time.Duration
err error
type peerResponse struct {
flapCount int
ts *time.Time
err error
}
// NewChannelEventStore initializes an event store with the config provided.
@@ -111,10 +128,9 @@ type uptimeResponse struct {
func NewChannelEventStore(config *Config) *ChannelEventStore {
store := &ChannelEventStore{
cfg: config,
channels: make(map[wire.OutPoint]*chanEventLog),
peers: make(map[route.Vertex]bool),
lifespanRequests: make(chan lifespanRequest),
uptimeRequests: make(chan uptimeRequest),
peers: make(map[route.Vertex]peerMonitor),
chanInfoRequests: make(chan channelInfoRequest),
peerRequests: make(chan peerRequest),
quit: make(chan struct{}),
}
@@ -140,7 +156,8 @@ func (c *ChannelEventStore) Start() error {
return err
}
// cancel should be called to cancel all subscriptions if an error occurs.
// cancel should be called to cancel all subscriptions if an error
// occurs.
cancel := func() {
channelClient.Cancel()
peerClient.Cancel()
@@ -166,8 +183,8 @@ func (c *ChannelEventStore) Start() error {
return err
}
// Add existing channels to the channel store with an initial peer
// online or offline event.
// Add existing channels to the channel store with an initial
// peer online or offline event.
c.addChannel(ch.FundingOutpoint, peerKey)
}
@@ -186,63 +203,98 @@ func (c *ChannelEventStore) Start() error {
func (c *ChannelEventStore) Stop() {
log.Info("Stopping event store")
c.cfg.FlapCountTicker.Stop()
// Stop the consume goroutine.
close(c.quit)
c.wg.Wait()
}
// addChannel adds a new channel to the ChannelEventStore's map of channels with
// an initial peer online state (if the peer is online). If the channel is
// already present in the map, the function returns early. This function should
// be called to add existing channels on startup and when open channel events
// are observed.
// addChannel checks whether we are already tracking a channel's peer, creates a
// new peer log to track it if we are not yet monitoring it, and adds the
// channel.
func (c *ChannelEventStore) addChannel(channelPoint wire.OutPoint,
peer route.Vertex) {
// Check for the unexpected case where the channel is already in the store.
_, ok := c.channels[channelPoint]
if ok {
log.Errorf("Channel %v duplicated in channel store", channelPoint)
peerMonitor, err := c.getPeerMonitor(peer)
if err != nil {
log.Error("could not create monitor: %v", err)
return
}
// Create an event log for the channel.
eventLog := newEventLog(channelPoint, peer, time.Now)
if err := peerMonitor.addChannel(channelPoint); err != nil {
log.Errorf("could not add channel: %v", err)
}
}
// If the peer is already online, add a peer online event to record
// the starting state of the peer.
if c.peers[peer] {
eventLog.add(peerOnlineEvent)
// getPeerMonitor tries to get an existing peer monitor from our in memory list,
// and falls back to creating a new monitor if it is not currently known.
func (c *ChannelEventStore) getPeerMonitor(peer route.Vertex) (peerMonitor,
error) {
peerMonitor, ok := c.peers[peer]
if ok {
return peerMonitor, nil
}
c.channels[channelPoint] = eventLog
var (
flapCount int
lastFlap *time.Time
)
historicalFlap, err := c.cfg.ReadFlapCount(peer)
switch err {
// If we do not have any records for this peer we set a 0 flap count
// and timestamp.
case channeldb.ErrNoPeerBucket:
case nil:
flapCount = int(historicalFlap.Count)
lastFlap = &historicalFlap.LastFlap
// Return if we get an unexpected error.
default:
return nil, err
}
peerMonitor = newPeerLog(c.cfg.Clock, flapCount, lastFlap)
c.peers[peer] = peerMonitor
return peerMonitor, nil
}
// closeChannel records a closed time for a channel, and returns early is the
// channel is not known to the event store.
func (c *ChannelEventStore) closeChannel(channelPoint wire.OutPoint) {
// Check for the unexpected case where the channel is unknown to the store.
eventLog, ok := c.channels[channelPoint]
// channel is not known to the event store. We log warnings (rather than errors)
// when we cannot find a peer/channel because channels that we restore from a
// static channel backup do not have their open notified, so the event store
// never learns about them, but they are closed using the regular flow so we
// will try to remove them on close. At present, we cannot easily distinguish
// between these closes and others.
func (c *ChannelEventStore) closeChannel(channelPoint wire.OutPoint,
peer route.Vertex) {
peerMonitor, ok := c.peers[peer]
if !ok {
log.Errorf("Close channel %v unknown to store", channelPoint)
log.Warnf("peer not known to store: %v", peer)
return
}
eventLog.close()
if err := peerMonitor.removeChannel(channelPoint); err != nil {
log.Warnf("could not remove channel: %v", err)
}
}
// peerEvent adds a peer online or offline event to all channels we currently
// have open with a peer.
func (c *ChannelEventStore) peerEvent(peer route.Vertex, event eventType) {
// Track current online status of peers in the channelEventStore.
c.peers[peer] = event == peerOnlineEvent
for _, eventLog := range c.channels {
if eventLog.peer == peer {
eventLog.add(event)
}
// peerEvent creates a peer monitor for a peer if we do not currently have
// one, and adds an online event to it.
func (c *ChannelEventStore) peerEvent(peer route.Vertex, online bool) {
peerMonitor, err := c.getPeerMonitor(peer)
if err != nil {
log.Error("could not create monitor: %v", err)
return
}
peerMonitor.onlineEvent(online)
}
// subscriptions abstracts away from subscription clients to allow for mocking.
@@ -256,8 +308,22 @@ type subscriptions struct {
// the event store with channel and peer events, and serves requests for channel
// uptime and lifespan.
func (c *ChannelEventStore) consume(subscriptions *subscriptions) {
defer c.wg.Done()
defer subscriptions.cancel()
// Start our flap count ticker.
c.cfg.FlapCountTicker.Resume()
// On exit, we will cancel our subscriptions and write our most recent
// flap counts to disk. This ensures that we have consistent data in
// the case of a graceful shutdown. If we do not shutdown gracefully,
// our worst case is data from our last flap count tick (1H).
defer func() {
subscriptions.cancel()
if err := c.recordFlapCount(); err != nil {
log.Errorf("error recording flap on shutdown: %v", err)
}
c.wg.Done()
}()
// Consume events until the channel is closed.
for {
@@ -265,68 +331,78 @@ func (c *ChannelEventStore) consume(subscriptions *subscriptions) {
// Process channel opened and closed events.
case e := <-subscriptions.channelUpdates:
switch event := e.(type) {
// A new channel has been opened, we must add the channel to the
// store and record a channel open event.
// A new channel has been opened, we must add the
// channel to the store and record a channel open event.
case channelnotifier.OpenChannelEvent:
compressed := event.Channel.IdentityPub.SerializeCompressed()
peerKey, err := route.NewVertexFromBytes(
event.Channel.IdentityPub.SerializeCompressed(),
compressed,
)
if err != nil {
log.Errorf("Could not get vertex from: %v",
event.Channel.IdentityPub.SerializeCompressed())
log.Errorf("Could not get vertex "+
"from: %v", compressed)
}
c.addChannel(event.Channel.FundingOutpoint, peerKey)
c.addChannel(
event.Channel.FundingOutpoint, peerKey,
)
// A channel has been closed, we must remove the channel from the
// store and record a channel closed event.
// A channel has been closed, we must remove the channel
// from the store and record a channel closed event.
case channelnotifier.ClosedChannelEvent:
c.closeChannel(event.CloseSummary.ChanPoint)
compressed := event.CloseSummary.RemotePub.SerializeCompressed()
peerKey, err := route.NewVertexFromBytes(
compressed,
)
if err != nil {
log.Errorf("Could not get vertex "+
"from: %v", compressed)
continue
}
c.closeChannel(
event.CloseSummary.ChanPoint, peerKey,
)
}
// Process peer online and offline events.
case e := <-subscriptions.peerUpdates:
switch event := e.(type) {
// We have reestablished a connection with our peer, and should
// record an online event for any channels with that peer.
// We have reestablished a connection with our peer,
// and should record an online event for any channels
// with that peer.
case peernotifier.PeerOnlineEvent:
c.peerEvent(event.PubKey, peerOnlineEvent)
c.peerEvent(event.PubKey, true)
// We have lost a connection with our peer, and should record an
// offline event for any channels with that peer.
// We have lost a connection with our peer, and should
// record an offline event for any channels with that
// peer.
case peernotifier.PeerOfflineEvent:
c.peerEvent(event.PubKey, peerOfflineEvent)
c.peerEvent(event.PubKey, false)
}
// Serve all requests for channel lifetime.
case req := <-c.lifespanRequests:
var resp lifespanResponse
channel, ok := c.channels[req.channelPoint]
if !ok {
resp.err = ErrChannelNotFound
} else {
resp.start = channel.openedAt
resp.end = channel.closedAt
}
case req := <-c.chanInfoRequests:
var resp channelInfoResponse
resp.info, resp.err = c.getChanInfo(req)
req.responseChan <- resp
// Serve requests for channel uptime.
case req := <-c.uptimeRequests:
var resp uptimeResponse
channel, ok := c.channels[req.channelPoint]
if !ok {
resp.err = ErrChannelNotFound
} else {
uptime, err := channel.uptime(req.startTime, req.endTime)
resp.uptime = uptime
resp.err = err
}
// Serve all requests for information about our peer.
case req := <-c.peerRequests:
var resp peerResponse
resp.flapCount, resp.ts, resp.err = c.flapCount(
req.peer,
)
req.responseChan <- resp
case <-c.cfg.FlapCountTicker.Ticks():
if err := c.recordFlapCount(); err != nil {
log.Errorf("could not record flap "+
"count: %v", err)
}
// Exit if the store receives the signal to shutdown.
case <-c.quit:
return
@@ -334,65 +410,151 @@ func (c *ChannelEventStore) consume(subscriptions *subscriptions) {
}
}
// GetLifespan returns the opening and closing time observed for a channel and
// a boolean to indicate whether the channel is known the the event store. If
// the channel is still open, a zero close time is returned.
func (c *ChannelEventStore) GetLifespan(
channelPoint wire.OutPoint) (time.Time, time.Time, error) {
// ChannelInfo provides the set of information that the event store has recorded
// for a channel.
type ChannelInfo struct {
// Lifetime is the total amount of time we have monitored the channel
// for.
Lifetime time.Duration
request := lifespanRequest{
// Uptime is the total amount of time that the channel peer has been
// observed as online during the monitored lifespan.
Uptime time.Duration
}
// GetChanInfo gets all the information we have on a channel in the event store.
func (c *ChannelEventStore) GetChanInfo(channelPoint wire.OutPoint,
peer route.Vertex) (*ChannelInfo, error) {
request := channelInfoRequest{
peer: peer,
channelPoint: channelPoint,
responseChan: make(chan lifespanResponse),
responseChan: make(chan channelInfoResponse),
}
// Send a request for the channel's lifespan to the main event loop, or
// return early with an error if the store has already received a shutdown
// signal.
// Send a request for the channel's information to the main event loop,
// or return early with an error if the store has already received a
// shutdown signal.
select {
case c.lifespanRequests <- request:
case c.chanInfoRequests <- request:
case <-c.quit:
return time.Time{}, time.Time{}, errShuttingDown
return nil, errShuttingDown
}
// Return the response we receive on the response channel or exit early if
// the store is instructed to exit.
// Return the response we receive on the response channel or exit early
// if the store is instructed to exit.
select {
case resp := <-request.responseChan:
return resp.start, resp.end, resp.err
return resp.info, resp.err
case <-c.quit:
return time.Time{}, time.Time{}, errShuttingDown
return nil, errShuttingDown
}
}
// GetUptime returns the uptime of a channel over a period and an error if the
// channel cannot be found or the uptime calculation fails.
func (c *ChannelEventStore) GetUptime(channelPoint wire.OutPoint, startTime,
endTime time.Time) (time.Duration, error) {
// getChanInfo collects channel information for a channel. It gets uptime over
// the full lifetime of the channel.
func (c *ChannelEventStore) getChanInfo(req channelInfoRequest) (*ChannelInfo,
error) {
request := uptimeRequest{
channelPoint: channelPoint,
startTime: startTime,
endTime: endTime,
responseChan: make(chan uptimeResponse),
peerMonitor, ok := c.peers[req.peer]
if !ok {
return nil, ErrPeerNotFound
}
// Send a request for the channel's uptime to the main event loop, or
// return early with an error if the store has already received a shutdown
// signal.
lifetime, uptime, err := peerMonitor.channelUptime(req.channelPoint)
if err != nil {
return nil, err
}
return &ChannelInfo{
Lifetime: lifetime,
Uptime: uptime,
}, nil
}
// FlapCount returns the flap count we have for a peer and the timestamp of its
// last flap. If we do not have any flaps recorded for the peer, the last flap
// timestamp will be nil.
func (c *ChannelEventStore) FlapCount(peer route.Vertex) (int, *time.Time,
error) {
request := peerRequest{
peer: peer,
responseChan: make(chan peerResponse),
}
// Send a request for the peer's information to the main event loop,
// or return early with an error if the store has already received a
// shutdown signal.
select {
case c.uptimeRequests <- request:
case c.peerRequests <- request:
case <-c.quit:
return 0, errShuttingDown
return 0, nil, errShuttingDown
}
// Return the response we receive on the response channel or exit early if
// the store is instructed to exit.
// Return the response we receive on the response channel or exit early
// if the store is instructed to exit.
select {
case resp := <-request.responseChan:
return resp.uptime, resp.err
return resp.flapCount, resp.ts, resp.err
case <-c.quit:
return 0, errShuttingDown
return 0, nil, errShuttingDown
}
}
// flapCount gets our peer flap count and last flap timestamp from our in memory
// record of a peer, falling back to on disk if we are not currently tracking
// the peer. If we have no flap count recorded for the peer, a nil last flap
// time will be returned.
func (c *ChannelEventStore) flapCount(peer route.Vertex) (int, *time.Time,
error) {
// First check whether we are tracking this peer in memory, because this
// record will have the most accurate flap count. We do not fail if we
// can't find the peer in memory, because we may have previously
// recorded its flap count on disk.
peerMonitor, ok := c.peers[peer]
if ok {
count, ts := peerMonitor.getFlapCount()
return count, ts, nil
}
// Try to get our flap count from the database. If this value is not
// recorded, we return a nil last flap time to indicate that we have no
// record of the peer's flap count.
flapCount, err := c.cfg.ReadFlapCount(peer)
switch err {
case channeldb.ErrNoPeerBucket:
return 0, nil, nil
case nil:
return int(flapCount.Count), &flapCount.LastFlap, nil
default:
return 0, nil, err
}
}
// recordFlapCount will record our flap count for each peer that we are
// currently tracking, skipping peers that have a 0 flap count.
func (c *ChannelEventStore) recordFlapCount() error {
updates := make(peerFlapCountMap)
for peer, monitor := range c.peers {
flapCount, lastFlap := monitor.getFlapCount()
if lastFlap == nil {
continue
}
updates[peer] = &channeldb.FlapCount{
Count: uint32(flapCount),
LastFlap: *lastFlap,
}
}
log.Debugf("recording flap count for: %v peers", len(updates))
return c.cfg.WriteFlapCount(updates)
}
+232 -418
View File
@@ -2,38 +2,40 @@ package chanfitness
import (
"errors"
"math/big"
"testing"
"time"
"github.com/btcsuite/btcd/btcec"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channelnotifier"
"github.com/lightningnetwork/lnd/peernotifier"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/routing/route"
"github.com/lightningnetwork/lnd/subscribe"
"github.com/stretchr/testify/require"
)
// testNow is the current time tests will use.
var testNow = time.Unix(1592465134, 0)
// TestStartStoreError tests the starting of the store in cases where the setup
// functions fail. It does not test the mechanics of consuming events because
// these are covered in a separate set of tests.
func TestStartStoreError(t *testing.T) {
// Ok and erroring subscribe functions are defined here to de-clutter tests.
okSubscribeFunc := func() (*subscribe.Client, error) {
return &subscribe.Client{
Cancel: func() {},
}, nil
// Ok and erroring subscribe functions are defined here to de-clutter
// tests.
okSubscribeFunc := func() (subscribe.Subscription, error) {
return newMockSubscription(t), nil
}
errSubscribeFunc := func() (client *subscribe.Client, e error) {
errSubscribeFunc := func() (subscribe.Subscription, error) {
return nil, errors.New("intentional test err")
}
tests := []struct {
name string
ChannelEvents func() (*subscribe.Client, error)
PeerEvents func() (*subscribe.Client, error)
ChannelEvents func() (subscribe.Subscription, error)
PeerEvents func() (subscribe.Subscription, error)
GetChannels func() ([]*channeldb.OpenChannel, error)
}{
{
@@ -49,7 +51,7 @@ func TestStartStoreError(t *testing.T) {
name: "Get open channels fails",
ChannelEvents: okSubscribeFunc,
PeerEvents: okSubscribeFunc,
GetChannels: func() (channels []*channeldb.OpenChannel, e error) {
GetChannels: func() ([]*channeldb.OpenChannel, error) {
return nil, errors.New("intentional test err")
},
},
@@ -59,15 +61,18 @@ func TestStartStoreError(t *testing.T) {
test := test
t.Run(test.name, func(t *testing.T) {
clock := clock.NewTestClock(testNow)
store := NewChannelEventStore(&Config{
SubscribeChannelEvents: test.ChannelEvents,
SubscribePeerEvents: test.PeerEvents,
GetOpenChannels: test.GetChannels,
Clock: clock,
})
err := store.Start()
// Check that we receive an error, because the test only checks for
// error cases.
// Check that we receive an error, because the test only
// checks for error cases.
if err == nil {
t.Fatalf("Expected error on startup, got: nil")
}
@@ -75,29 +80,6 @@ func TestStartStoreError(t *testing.T) {
}
}
// getTestChannel returns a non-zero peer pubKey, serialized pubKey and channel
// outpoint for testing.
func getTestChannel(t *testing.T) (*btcec.PublicKey, route.Vertex,
wire.OutPoint) {
privKey, err := btcec.NewPrivateKey(btcec.S256())
if err != nil {
t.Fatalf("Error getting pubkey: %v", err)
}
pubKey, err := route.NewVertexFromBytes(
privKey.PubKey().SerializeCompressed(),
)
if err != nil {
t.Fatalf("Could not create vertex: %v", err)
}
return privKey.PubKey(), pubKey, wire.OutPoint{
Hash: [chainhash.HashSize]byte{1, 2, 3},
Index: 0,
}
}
// TestMonitorChannelEvents tests the store's handling of channel and peer
// events. It tests for the unexpected cases where we receive a channel open for
// an already known channel and but does not test for closing an unknown channel
@@ -105,425 +87,257 @@ func getTestChannel(t *testing.T) (*btcec.PublicKey, route.Vertex,
// through an eventLog which does not exist. This test does not test handling
// of uptime and lifespan requests, as they are tested in their own tests.
func TestMonitorChannelEvents(t *testing.T) {
pubKey, vertex, chanPoint := getTestChannel(t)
var (
pubKey = &btcec.PublicKey{
X: big.NewInt(0),
Y: big.NewInt(1),
Curve: btcec.S256(),
}
tests := []struct {
name string
chan1 = wire.OutPoint{Index: 1}
chan2 = wire.OutPoint{Index: 2}
)
// generateEvents takes channels which represent the updates channels
// for subscription clients and passes events in the desired order.
// This function is intended to be blocking so that the test does not
// have a data race with event consumption, so the channels should not
// be buffered.
generateEvents func(channelEvents, peerEvents chan<- interface{})
peer1, err := route.NewVertexFromBytes(pubKey.SerializeCompressed())
require.NoError(t, err)
// expectedEvents is the expected set of event types in the store.
expectedEvents []eventType
}{
{
name: "Channel opened, peer comes online",
generateEvents: func(channelEvents, peerEvents chan<- interface{}) {
// Add an open channel event
channelEvents <- channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: chanPoint,
IdentityPub: pubKey,
},
}
t.Run("peer comes online after channel open", func(t *testing.T) {
gen := func(ctx *chanEventStoreTestCtx) {
ctx.sendChannelOpenedUpdate(pubKey, chan1)
ctx.peerEvent(peer1, true)
}
// Add a peer online event.
peerEvents <- peernotifier.PeerOnlineEvent{PubKey: vertex}
},
expectedEvents: []eventType{peerOnlineEvent},
},
{
name: "Duplicate channel open events",
generateEvents: func(channelEvents, peerEvents chan<- interface{}) {
// Add an open channel event
channelEvents <- channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: chanPoint,
IdentityPub: pubKey,
},
}
testEventStore(t, gen, peer1, 1)
})
// Add a peer online event.
peerEvents <- peernotifier.PeerOnlineEvent{PubKey: vertex}
t.Run("duplicate channel open events", func(t *testing.T) {
gen := func(ctx *chanEventStoreTestCtx) {
ctx.sendChannelOpenedUpdate(pubKey, chan1)
ctx.sendChannelOpenedUpdate(pubKey, chan1)
ctx.peerEvent(peer1, true)
}
// Add a duplicate channel open event.
channelEvents <- channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: chanPoint,
IdentityPub: pubKey,
},
}
},
expectedEvents: []eventType{peerOnlineEvent},
},
{
name: "Channel opened, peer already online",
generateEvents: func(channelEvents, peerEvents chan<- interface{}) {
// Add a peer online event.
peerEvents <- peernotifier.PeerOnlineEvent{PubKey: vertex}
testEventStore(t, gen, peer1, 1)
})
// Add an open channel event
channelEvents <- channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: chanPoint,
IdentityPub: pubKey,
},
}
},
expectedEvents: []eventType{peerOnlineEvent},
},
t.Run("peer online before channel created", func(t *testing.T) {
gen := func(ctx *chanEventStoreTestCtx) {
ctx.peerEvent(peer1, true)
ctx.sendChannelOpenedUpdate(pubKey, chan1)
}
{
name: "Channel opened, peer offline, closed",
generateEvents: func(channelEvents, peerEvents chan<- interface{}) {
// Add an open channel event
channelEvents <- channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: chanPoint,
IdentityPub: pubKey,
},
}
testEventStore(t, gen, peer1, 1)
})
// Add a peer online event.
peerEvents <- peernotifier.PeerOfflineEvent{PubKey: vertex}
t.Run("multiple channels for peer", func(t *testing.T) {
gen := func(ctx *chanEventStoreTestCtx) {
ctx.peerEvent(peer1, true)
ctx.sendChannelOpenedUpdate(pubKey, chan1)
// Add a close channel event.
channelEvents <- channelnotifier.ClosedChannelEvent{
CloseSummary: &channeldb.ChannelCloseSummary{
ChanPoint: chanPoint,
},
}
},
expectedEvents: []eventType{peerOfflineEvent},
},
{
name: "Event after channel close not recorded",
generateEvents: func(channelEvents, peerEvents chan<- interface{}) {
// Add an open channel event
channelEvents <- channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: chanPoint,
IdentityPub: pubKey,
},
}
ctx.peerEvent(peer1, false)
ctx.sendChannelOpenedUpdate(pubKey, chan2)
}
// Add a close channel event.
channelEvents <- channelnotifier.ClosedChannelEvent{
CloseSummary: &channeldb.ChannelCloseSummary{
ChanPoint: chanPoint,
},
}
testEventStore(t, gen, peer1, 2)
})
// Add a peer online event.
peerEvents <- peernotifier.PeerOfflineEvent{PubKey: vertex}
},
},
}
t.Run("multiple channels for peer, one closed", func(t *testing.T) {
gen := func(ctx *chanEventStoreTestCtx) {
ctx.peerEvent(peer1, true)
ctx.sendChannelOpenedUpdate(pubKey, chan1)
for _, test := range tests {
test := test
ctx.peerEvent(peer1, false)
ctx.sendChannelOpenedUpdate(pubKey, chan2)
t.Run(test.name, func(t *testing.T) {
// Create a store with the channels and online peers specified
// by the test.
store := NewChannelEventStore(&Config{})
ctx.closeChannel(chan1, pubKey)
ctx.peerEvent(peer1, true)
}
// Create channels which represent the subscriptions we have to peer
// and client events.
channelEvents := make(chan interface{})
peerEvents := make(chan interface{})
testEventStore(t, gen, peer1, 1)
})
store.wg.Add(1)
go store.consume(&subscriptions{
channelUpdates: channelEvents,
peerUpdates: peerEvents,
cancel: func() {},
})
// Add events to the store then kill the goroutine using store.Stop.
test.generateEvents(channelEvents, peerEvents)
store.Stop()
// Retrieve the eventLog for the channel and check that its
// contents are as expected.
eventLog, ok := store.channels[chanPoint]
if !ok {
t.Fatalf("Expected to find event store")
}
for i, e := range eventLog.events {
if test.expectedEvents[i] != e.eventType {
t.Fatalf("Expected type: %v, got: %v",
test.expectedEvents[i], e.eventType)
}
}
})
}
}
// TestGetLifetime tests the GetLifetime function for the cases where a channel
// testEventStore creates a new test contexts, generates a set of events for it
// and tests that it has the number of channels we expect.
func testEventStore(t *testing.T, generateEvents func(*chanEventStoreTestCtx),
peer route.Vertex, expectedChannels int) {
testCtx := newChanEventStoreTestCtx(t)
testCtx.start()
generateEvents(testCtx)
// Shutdown the store so that we can safely access the maps in our event
// store.
testCtx.stop()
// Get our peer and check that it has the channels we expect.
monitor, ok := testCtx.store.peers[peer]
require.True(t, ok)
require.Equal(t, expectedChannels, monitor.channelCount())
}
// TestStoreFlapCount tests flushing of flap counts to disk on timer ticks and
// on store shutdown.
func TestStoreFlapCount(t *testing.T) {
testCtx := newChanEventStoreTestCtx(t)
testCtx.start()
pubkey, _, _ := testCtx.createChannel()
testCtx.peerEvent(pubkey, false)
// Now, we tick our flap count ticker. We expect our main goroutine to
// flush our tick count to disk.
testCtx.tickFlapCount()
// Since we just tracked a offline event, we expect a single flap for
// our peer.
expectedUpdate := peerFlapCountMap{
pubkey: {
Count: 1,
LastFlap: testCtx.clock.Now(),
},
}
testCtx.assertFlapCountUpdated()
testCtx.assertFlapCountUpdates(expectedUpdate)
// Create three events for out peer, online/offline/online.
testCtx.peerEvent(pubkey, true)
testCtx.peerEvent(pubkey, false)
testCtx.peerEvent(pubkey, true)
// Trigger another write.
testCtx.tickFlapCount()
// Since we have processed 3 more events for our peer, we update our
// expected online map to have a flap count of 4 for this peer.
expectedUpdate[pubkey] = &channeldb.FlapCount{
Count: 4,
LastFlap: testCtx.clock.Now(),
}
testCtx.assertFlapCountUpdated()
testCtx.assertFlapCountUpdates(expectedUpdate)
testCtx.stop()
}
// TestGetChanInfo tests the GetChanInfo function for the cases where a channel
// is known and unknown to the store.
func TestGetLifetime(t *testing.T) {
now := time.Now()
func TestGetChanInfo(t *testing.T) {
ctx := newChanEventStoreTestCtx(t)
ctx.start()
tests := []struct {
name string
channelFound bool
channelPoint wire.OutPoint
opened time.Time
closed time.Time
expectedError error
}{
{
name: "Channel found",
channelFound: true,
opened: now,
closed: now.Add(time.Hour * -1),
expectedError: nil,
},
{
name: "Channel not found",
expectedError: ErrChannelNotFound,
},
}
// Make a note of the time that our mocked clock starts on.
now := ctx.clock.Now()
for _, test := range tests {
test := test
// Create mock vars for a channel but do not add them to our store yet.
peer, pk, channel := ctx.newChannel()
t.Run(test.name, func(t *testing.T) {
// Create and empty events store for testing.
store := NewChannelEventStore(&Config{})
// Send an online event for our peer, although we do not yet have an
// open channel.
ctx.peerEvent(peer, true)
// Start goroutine which consumes GetLifespan requests.
store.wg.Add(1)
go store.consume(&subscriptions{
channelUpdates: make(chan interface{}),
peerUpdates: make(chan interface{}),
cancel: func() {},
})
// Try to get info for a channel that has not been opened yet, we
// expect to get an error.
_, err := ctx.store.GetChanInfo(channel, peer)
require.Equal(t, ErrChannelNotFound, err)
// Stop the store's go routine.
defer store.Stop()
// Now we send our store a notification that a channel has been opened.
ctx.sendChannelOpenedUpdate(pk, channel)
// Add channel to eventStore if the test indicates that it should
// be present.
if test.channelFound {
store.channels[test.channelPoint] = &chanEventLog{
openedAt: test.opened,
closedAt: test.closed,
}
}
// Wait for our channel to be recognized by our store. We need to wait
// for the channel to be created so that we do not update our time
// before the channel open is processed.
require.Eventually(t, func() bool {
_, err = ctx.store.GetChanInfo(channel, peer)
return err == nil
}, timeout, time.Millisecond*20)
open, close, err := store.GetLifespan(test.channelPoint)
if test.expectedError != err {
t.Fatalf("Expected: %v, got: %v", test.expectedError, err)
}
// Increment our test clock by an hour.
now = now.Add(time.Hour)
ctx.clock.SetTime(now)
if open != test.opened {
t.Errorf("Expected: %v, got %v", test.opened, open)
}
// At this stage our channel has been open and online for an hour.
info, err := ctx.store.GetChanInfo(channel, peer)
require.NoError(t, err)
require.Equal(t, time.Hour, info.Lifetime)
require.Equal(t, time.Hour, info.Uptime)
if close != test.closed {
t.Errorf("Expected: %v, got %v", test.closed, close)
}
})
}
// Now we send a peer offline event for our channel.
ctx.peerEvent(peer, false)
// Since we have not bumped our mocked time, our uptime calculations
// should be the same, even though we've just processed an offline
// event.
info, err = ctx.store.GetChanInfo(channel, peer)
require.NoError(t, err)
require.Equal(t, time.Hour, info.Lifetime)
require.Equal(t, time.Hour, info.Uptime)
// Progress our time again. This time, our peer is currently tracked as
// being offline, so we expect our channel info to reflect that the peer
// has been offline for this period.
now = now.Add(time.Hour)
ctx.clock.SetTime(now)
info, err = ctx.store.GetChanInfo(channel, peer)
require.NoError(t, err)
require.Equal(t, time.Hour*2, info.Lifetime)
require.Equal(t, time.Hour, info.Uptime)
ctx.stop()
}
// TestGetUptime tests the getUptime call for channels known to the event store.
// It does not test the trivial case where a channel is unknown to the store,
// because this is simply a zero return if an item is not found in a map. It
// tests the unexpected edge cases where a tracked channel does not have any
// events recorded, and when a zero time is specified for the uptime range.
func TestGetUptime(t *testing.T) {
// Set time for deterministic unit tests.
now := time.Now()
// TestFlapCount tests querying the store for peer flap counts, covering the
// case where the peer is tracked in memory, and the case where we need to
// lookup the peer on disk.
func TestFlapCount(t *testing.T) {
clock := clock.NewTestClock(testNow)
twoHoursAgo := now.Add(time.Hour * -2)
fourHoursAgo := now.Add(time.Hour * -4)
var (
peer = route.Vertex{9, 9, 9}
peerFlapCount = 3
lastFlap = clock.Now()
)
tests := []struct {
name string
channelPoint wire.OutPoint
// events is the set of events we expect to find in the channel store.
events []*channelEvent
// openedAt is the time the channel is recorded as open by the store.
openedAt time.Time
// closedAt is the time the channel is recorded as closed by the store.
// If the channel is still open, this value is zero.
closedAt time.Time
// channelFound is true if we expect to find the channel in the store.
channelFound bool
// startTime specifies the beginning of the uptime range we want to
// calculate.
startTime time.Time
// endTime specified the end of the uptime range we want to calculate.
endTime time.Time
expectedUptime time.Duration
expectedError error
}{
{
name: "No events",
startTime: twoHoursAgo,
endTime: now,
channelFound: true,
expectedError: nil,
},
{
name: "50% Uptime",
events: []*channelEvent{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
{
timestamp: twoHoursAgo,
eventType: peerOfflineEvent,
},
},
openedAt: fourHoursAgo,
expectedUptime: time.Hour * 2,
startTime: fourHoursAgo,
endTime: now,
channelFound: true,
expectedError: nil,
},
{
name: "Zero start time",
events: []*channelEvent{
{
timestamp: fourHoursAgo,
eventType: peerOnlineEvent,
},
},
openedAt: fourHoursAgo,
expectedUptime: time.Hour * 4,
endTime: now,
channelFound: true,
expectedError: nil,
},
{
name: "Channel not found",
startTime: twoHoursAgo,
endTime: now,
channelFound: false,
expectedError: ErrChannelNotFound,
},
// Create a test context with one peer's flap count already recorded,
// which mocks it already having its flap count stored on disk.
ctx := newChanEventStoreTestCtx(t)
ctx.flapUpdates[peer] = &channeldb.FlapCount{
Count: uint32(peerFlapCount),
LastFlap: lastFlap,
}
for _, test := range tests {
test := test
ctx.start()
t.Run(test.name, func(t *testing.T) {
// Set up event store with the events specified for the test and
// mocked time.
store := NewChannelEventStore(&Config{})
// Create test variables for a peer and channel, but do not add it to
// our store yet.
peer1 := route.Vertex{1, 2, 3}
// Start goroutine which consumes GetUptime requests.
store.wg.Add(1)
go store.consume(&subscriptions{
channelUpdates: make(chan interface{}),
peerUpdates: make(chan interface{}),
cancel: func() {},
})
// First, query for a peer that we have no record of in memory or on
// disk and confirm that we indicate that the peer was not found.
_, ts, err := ctx.store.FlapCount(peer1)
require.NoError(t, err)
require.Nil(t, ts)
// Stop the store's goroutine.
defer store.Stop()
// Send an online event for our peer.
ctx.peerEvent(peer1, true)
// Add the channel to the store if it is intended to be found.
if test.channelFound {
store.channels[test.channelPoint] = &chanEventLog{
events: test.events,
now: func() time.Time { return now },
openedAt: test.openedAt,
closedAt: test.closedAt,
}
}
// Assert that we now find a record of the peer with flap count = 1.
count, ts, err := ctx.store.FlapCount(peer1)
require.NoError(t, err)
require.Equal(t, lastFlap, *ts)
require.Equal(t, 1, count)
uptime, err := store.GetUptime(test.channelPoint, test.startTime, test.endTime)
if test.expectedError != err {
t.Fatalf("Expected: %v, got: %v", test.expectedError, err)
}
// Make a request for our peer that not tracked in memory, but does
// have its flap count stored on disk.
count, ts, err = ctx.store.FlapCount(peer)
require.NoError(t, err)
require.Equal(t, lastFlap, *ts)
require.Equal(t, peerFlapCount, count)
if uptime != test.expectedUptime {
t.Fatalf("Expected uptime percentage: %v, got %v",
test.expectedUptime, uptime)
}
})
}
}
// TestAddChannel tests that channels are added to the event store with
// appropriate timestamps. This test addresses a bug where offline channels
// did not have an opened time set, and checks that an online event is set for
// peers that are online at the time that a channel is opened.
func TestAddChannel(t *testing.T) {
_, vertex, chanPoint := getTestChannel(t)
tests := []struct {
name string
// peers maps peers to an online state.
peers map[route.Vertex]bool
expectedEvents []eventType
}{
{
name: "peer offline",
peers: make(map[route.Vertex]bool),
expectedEvents: []eventType{},
},
{
name: "peer online",
peers: map[route.Vertex]bool{
vertex: true,
},
expectedEvents: []eventType{peerOnlineEvent},
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
store := NewChannelEventStore(&Config{})
store.peers = test.peers
// Add channel to the store.
store.addChannel(chanPoint, vertex)
// Check that the eventLog is successfully added.
eventLog, ok := store.channels[chanPoint]
if !ok {
t.Fatalf("channel should be in store")
}
// Check that the eventLog contains the events we
// expect.
for i, e := range test.expectedEvents {
if e != eventLog.events[i].eventType {
t.Fatalf("expected: %v, got: %v",
e, eventLog.events[i].eventType)
}
}
// Ensure that open time is always set.
if eventLog.openedAt.IsZero() {
t.Fatalf("channel should have opened at set")
}
})
}
ctx.stop()
}
+306
View File
@@ -0,0 +1,306 @@
package chanfitness
import (
"math/big"
"testing"
"time"
"github.com/btcsuite/btcd/btcec"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channelnotifier"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/peernotifier"
"github.com/lightningnetwork/lnd/routing/route"
"github.com/lightningnetwork/lnd/subscribe"
"github.com/lightningnetwork/lnd/ticker"
"github.com/stretchr/testify/require"
)
// timeout is the amount of time we allow our blocking test calls.
var timeout = time.Second
// chanEventStoreTestCtx is a helper struct which can be used to test the
// channel event store.
type chanEventStoreTestCtx struct {
t *testing.T
store *ChannelEventStore
channelSubscription *mockSubscription
peerSubscription *mockSubscription
// testVarIdx is an index which will be used to deterministically add
// channels and public keys to our test context. We use a single value
// for a single pubkey + channel combination because its actual value
// does not matter.
testVarIdx int
// clock is the clock that our test store will use.
clock *clock.TestClock
// flapUpdates stores our most recent set of updates flap counts.
flapUpdates peerFlapCountMap
// flapCountUpdates is a channel which receives new flap counts.
flapCountUpdates chan peerFlapCountMap
// stopped is closed when our test context is fully shutdown. It is
// used to prevent calling of functions which can only be called after
// shutdown.
stopped chan struct{}
}
// newChanEventStoreTestCtx creates a test context which can be used to test
// the event store.
func newChanEventStoreTestCtx(t *testing.T) *chanEventStoreTestCtx {
testCtx := &chanEventStoreTestCtx{
t: t,
channelSubscription: newMockSubscription(t),
peerSubscription: newMockSubscription(t),
clock: clock.NewTestClock(testNow),
flapUpdates: make(peerFlapCountMap),
flapCountUpdates: make(chan peerFlapCountMap),
stopped: make(chan struct{}),
}
cfg := &Config{
Clock: testCtx.clock,
SubscribeChannelEvents: func() (subscribe.Subscription, error) {
return testCtx.channelSubscription, nil
},
SubscribePeerEvents: func() (subscribe.Subscription, error) {
return testCtx.peerSubscription, nil
},
GetOpenChannels: func() ([]*channeldb.OpenChannel, error) {
return nil, nil
},
WriteFlapCount: func(updates map[route.Vertex]*channeldb.FlapCount) error {
// Send our whole update map into the test context's
// updates channel. The test will need to assert flap
// count updated or this send will timeout.
select {
case testCtx.flapCountUpdates <- updates:
case <-time.After(timeout):
t.Fatalf("WriteFlapCount timeout")
}
return nil
},
ReadFlapCount: func(peer route.Vertex) (*channeldb.FlapCount, error) {
count, ok := testCtx.flapUpdates[peer]
if !ok {
return nil, channeldb.ErrNoPeerBucket
}
return count, nil
},
FlapCountTicker: ticker.NewForce(FlapCountFlushRate),
}
testCtx.store = NewChannelEventStore(cfg)
return testCtx
}
// start starts the test context's event store.
func (c *chanEventStoreTestCtx) start() {
require.NoError(c.t, c.store.Start())
}
// stop stops the channel event store's subscribe servers and the store itself.
func (c *chanEventStoreTestCtx) stop() {
// On shutdown of our event store, we write flap counts to disk. In our
// test context, this write function is blocked on asserting that the
// update has occurred. We stop our store in a goroutine so that we
// can shut it down and assert that it performs these on-shutdown
// updates. The stopped channel is used to ensure that we do not finish
// our test before this shutdown has completed.
go func() {
c.store.Stop()
close(c.stopped)
}()
// We write our flap count to disk on shutdown, assert that the most
// recent record that the server has is written on shutdown. Calling
// this assert unblocks the stop function above. We don't check values
// here, so that our tests don't all require providing an expected swap
// count, but at least assert that the write occurred.
c.assertFlapCountUpdated()
<-c.stopped
// Make sure that the cancel function was called for both of our
// subscription mocks.
c.channelSubscription.assertCancelled()
c.peerSubscription.assertCancelled()
}
// newChannel creates a new, unique test channel. Note that this function
// does not add it to the test event store, it just creates mocked values.
func (c *chanEventStoreTestCtx) newChannel() (route.Vertex, *btcec.PublicKey,
wire.OutPoint) {
// Create a pubkey for our channel peer.
pubKey := &btcec.PublicKey{
X: big.NewInt(int64(c.testVarIdx)),
Y: big.NewInt(int64(c.testVarIdx)),
Curve: btcec.S256(),
}
// Create vertex from our pubkey.
vertex, err := route.NewVertexFromBytes(pubKey.SerializeCompressed())
require.NoError(c.t, err)
// Create a channel point using our channel index, then increment it.
chanPoint := wire.OutPoint{
Hash: [chainhash.HashSize]byte{1, 2, 3},
Index: uint32(c.testVarIdx),
}
// Increment the index we use so that the next channel and pubkey we
// create will be unique.
c.testVarIdx++
return vertex, pubKey, chanPoint
}
// createChannel creates a new channel, notifies the event store that it has
// been created and returns the peer vertex, pubkey and channel point.
func (c *chanEventStoreTestCtx) createChannel() (route.Vertex, *btcec.PublicKey,
wire.OutPoint) {
vertex, pubKey, chanPoint := c.newChannel()
c.sendChannelOpenedUpdate(pubKey, chanPoint)
return vertex, pubKey, chanPoint
}
// closeChannel sends a close channel event to our subscribe server.
func (c *chanEventStoreTestCtx) closeChannel(channel wire.OutPoint,
peer *btcec.PublicKey) {
update := channelnotifier.ClosedChannelEvent{
CloseSummary: &channeldb.ChannelCloseSummary{
ChanPoint: channel,
RemotePub: peer,
},
}
c.channelSubscription.sendUpdate(update)
}
// tickFlapCount forces a tick for our flap count ticker with the current time.
func (c *chanEventStoreTestCtx) tickFlapCount() {
testTicker := c.store.cfg.FlapCountTicker.(*ticker.Force)
select {
case testTicker.Force <- c.store.cfg.Clock.Now():
case <-time.After(timeout):
c.t.Fatalf("could not tick flap count ticker")
}
}
// peerEvent sends a peer online or offline event to the store for the peer
// provided.
func (c *chanEventStoreTestCtx) peerEvent(peer route.Vertex, online bool) {
var update interface{}
if online {
update = peernotifier.PeerOnlineEvent{PubKey: peer}
} else {
update = peernotifier.PeerOfflineEvent{PubKey: peer}
}
c.peerSubscription.sendUpdate(update)
}
// sendChannelOpenedUpdate notifies the test event store that a channel has
// been opened.
func (c *chanEventStoreTestCtx) sendChannelOpenedUpdate(pubkey *btcec.PublicKey,
channel wire.OutPoint) {
update := channelnotifier.OpenChannelEvent{
Channel: &channeldb.OpenChannel{
FundingOutpoint: channel,
IdentityPub: pubkey,
},
}
c.channelSubscription.sendUpdate(update)
}
// assertFlapCountUpdated asserts that our store has made an attempt to write
// our current set of flap counts to disk and sets this value in our test ctx.
// Note that it does not check the values of the update.
func (c *chanEventStoreTestCtx) assertFlapCountUpdated() {
select {
case c.flapUpdates = <-c.flapCountUpdates:
case <-time.After(timeout):
c.t.Fatalf("assertFlapCountUpdated timeout")
}
}
// assertFlapCountUpdates asserts that out current record of flap counts is
// as expected.
func (c *chanEventStoreTestCtx) assertFlapCountUpdates(expected peerFlapCountMap) {
require.Equal(c.t, expected, c.flapUpdates)
}
// mockSubscription is a mock subscription client that blocks on sends into the
// updates channel. We use this mock rather than an actual subscribe client
// because they do not block, which makes tests race (because we have no way
// to guarantee that the test client consumes the update before shutdown).
type mockSubscription struct {
t *testing.T
updates chan interface{}
// Embed the subscription interface in this mock so that we satisfy it.
subscribe.Subscription
}
// newMockSubscription creates a mock subscription.
func newMockSubscription(t *testing.T) *mockSubscription {
return &mockSubscription{
t: t,
updates: make(chan interface{}),
}
}
// sendUpdate sends an update into our updates channel, mocking the dispatch of
// an update from a subscription server. This call will fail the test if the
// update is not consumed within our timeout.
func (m *mockSubscription) sendUpdate(update interface{}) {
select {
case m.updates <- update:
case <-time.After(timeout):
m.t.Fatalf("update: %v timeout", update)
}
}
// Updates returns the updates channel for the mock.
func (m *mockSubscription) Updates() <-chan interface{} {
return m.updates
}
// Cancel should be called in case the client no longer wants to subscribe for
// updates from the server.
func (m *mockSubscription) Cancel() {
close(m.updates)
}
// assertCancelled asserts that the cancel function has been called for this
// mock.
func (m *mockSubscription) assertCancelled() {
select {
case _, open := <-m.updates:
require.False(m.t, open, "subscription not cancelled")
case <-time.After(timeout):
m.t.Fatalf("assert cancelled timeout")
}
}
+34
View File
@@ -0,0 +1,34 @@
package chanfitness
import (
"time"
"github.com/btcsuite/btcd/wire"
)
// peerMonitor is an interface implemented by entities that monitor our peers
// online events and the channels we currently have open with them.
type peerMonitor interface {
// event adds an online or offline event.
onlineEvent(online bool)
// addChannel adds a new channel.
addChannel(channelPoint wire.OutPoint) error
// removeChannel removes a channel.
removeChannel(channelPoint wire.OutPoint) error
// channelCount returns the number of channels that we currently have
// with the peer.
channelCount() int
// channelUptime looks up a channel and returns the amount of time that
// the channel has been monitored for and its uptime over this period.
channelUptime(channelPoint wire.OutPoint) (time.Duration,
time.Duration, error)
// getFlapCount returns the peer's flap count and the timestamp that we
// last recorded a flap, which may be nil if we have never recorded a
// flap for this peer.
getFlapCount() (int, *time.Time)
}
+82
View File
@@ -0,0 +1,82 @@
package chanfitness
import (
"math"
"time"
)
const (
// rateLimitScale is the number of events we allow per rate limited
// tier. Increasing this value makes our rate limiting more lenient,
// decreasing it makes us less lenient.
rateLimitScale = 200
// flapCountCooldownFactor is the factor by which we decrease a peer's
// flap count if they have not flapped for the cooldown period.
flapCountCooldownFactor = 0.95
// flapCountCooldownPeriod is the amount of time that we require a peer
// has not flapped for before we reduce their all time flap count using
// our cooldown factor.
flapCountCooldownPeriod = time.Hour * 8
)
// rateLimits is the set of rate limit tiers we apply to our peers based on
// their flap count. A peer can be placed in their tier by dividing their flap
// count by the rateLimitScale and returning the value at that index.
var rateLimits = []time.Duration{
time.Second,
time.Second * 5,
time.Second * 30,
time.Minute,
time.Minute * 30,
time.Hour,
}
// getRateLimit returns the value of the rate limited tier that we are on based
// on current flap count. If a peer's flap count exceeds the top tier, we just
// return our highest tier.
func getRateLimit(flapCount int) time.Duration {
// Figure out the tier we fall into based on our current flap count.
tier := flapCount / rateLimitScale
// If we have more events than our number of tiers, we just use the
// last tier
tierLen := len(rateLimits)
if tier >= tierLen {
tier = tierLen - 1
}
return rateLimits[tier]
}
// cooldownFlapCount takes a timestamped flap count, and returns its value
// scaled down by our cooldown factor if at least our cooldown period has
// elapsed since the peer last flapped. We do this because we store all-time
// flap count for peers, and want to allow downgrading of peers that have not
// flapped for a long time.
func cooldownFlapCount(now time.Time, flapCount int,
lastFlap time.Time) int {
// Calculate time since our last flap, and the number of times we need
// to apply our cooldown factor.
timeSinceFlap := now.Sub(lastFlap)
// If our cooldown period has not elapsed yet, we just return our flap
// count. We allow fractional cooldown periods once this period has
// elapsed, so we do not want to apply a fractional cooldown before the
// full cooldown period has elapsed.
if timeSinceFlap < flapCountCooldownPeriod {
return flapCount
}
// Get the factor by which we need to cooldown our flap count. If
// insufficient time has passed to cooldown our flap count. Use use a
// float so that we allow fractional cooldown periods.
cooldownPeriods := float64(timeSinceFlap) /
float64(flapCountCooldownPeriod)
effectiveFactor := math.Pow(flapCountCooldownFactor, cooldownPeriods)
return int(float64(flapCount) * effectiveFactor)
}
+105
View File
@@ -0,0 +1,105 @@
package chanfitness
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestGetRateLimit tests getting of our rate limit using the current constants.
// It creates test cases that are relative to our constants so that they
// can be adjusted without breaking the unit test.
func TestGetRateLimit(t *testing.T) {
tests := []struct {
name string
flapCount int
rateLimit time.Duration
}{
{
name: "zero flaps",
flapCount: 0,
rateLimit: rateLimits[0],
},
{
name: "middle tier",
flapCount: rateLimitScale * (len(rateLimits) / 2),
rateLimit: rateLimits[len(rateLimits)/2],
},
{
name: "last tier",
flapCount: rateLimitScale * (len(rateLimits) - 1),
rateLimit: rateLimits[len(rateLimits)-1],
},
{
name: "beyond last tier",
flapCount: rateLimitScale * (len(rateLimits) * 2),
rateLimit: rateLimits[len(rateLimits)-1],
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
limit := getRateLimit(test.flapCount)
require.Equal(t, test.rateLimit, limit)
})
}
}
// TestCooldownFlapCount tests cooldown of all time flap counts.
func TestCooldownFlapCount(t *testing.T) {
tests := []struct {
name string
flapCount int
lastFlap time.Time
expected int
}{
{
name: "just flapped, do not cooldown",
flapCount: 1,
lastFlap: testNow,
expected: 1,
},
{
name: "period not elapsed, do not cooldown",
flapCount: 1,
lastFlap: testNow.Add(flapCountCooldownPeriod / 2 * -1),
expected: 1,
},
{
name: "rounded to 0",
flapCount: 1,
lastFlap: testNow.Add(flapCountCooldownPeriod * -1),
expected: 0,
},
{
name: "decreased to integer value",
flapCount: 10,
lastFlap: testNow.Add(flapCountCooldownPeriod * -1),
expected: 9,
},
{
name: "multiple cooldown periods",
flapCount: 10,
lastFlap: testNow.Add(flapCountCooldownPeriod * -3),
expected: 8,
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
t.Parallel()
flapCount := cooldownFlapCount(
testNow, test.flapCount, test.lastFlap,
)
require.Equal(t, test.expected, flapCount)
})
}
}
+104 -8
View File
@@ -79,6 +79,11 @@ var (
// for in one of our remote commits.
unsignedAckedUpdatesKey = []byte("unsigned-acked-updates-key")
// remoteUnsignedLocalUpdatesKey is an entry in the channel bucket that
// contains the local updates that the remote party has acked, but
// has not yet signed for in one of their local commits.
remoteUnsignedLocalUpdatesKey = []byte("remote-unsigned-local-updates-key")
// revocationStateKey stores their current revocation hash, our
// preimage producer and their preimage store.
revocationStateKey = []byte("revocation-state-key")
@@ -201,8 +206,7 @@ const (
// AnchorOutputsBit indicates that the channel makes use of anchor
// outputs to bump the commitment transaction's effective feerate. This
// channel type also uses a delayed to_remote output script. If bit is
// set, we'll find the size of the anchor outputs in the database.
// channel type also uses a delayed to_remote output script.
AnchorOutputsBit ChannelType = 1 << 3
// FrozenBit indicates that the channel is a frozen channel, meaning
@@ -1448,6 +1452,39 @@ func (c *OpenChannel) UpdateCommitment(newCommitment *ChannelCommitment,
"updates: %v", err)
}
// Persist the remote unsigned local updates that are not included
// in our new commitment.
updateBytes := chanBucket.Get(remoteUnsignedLocalUpdatesKey)
if updateBytes == nil {
return nil
}
r := bytes.NewReader(updateBytes)
updates, err := deserializeLogUpdates(r)
if err != nil {
return err
}
var validUpdates []LogUpdate
for _, upd := range updates {
// Filter for updates that are not on our local
// commitment.
if upd.LogIndex >= newCommitment.LocalLogIndex {
validUpdates = append(validUpdates, upd)
}
}
var b2 bytes.Buffer
err = serializeLogUpdates(&b2, validUpdates)
if err != nil {
return fmt.Errorf("unable to serialize log updates: %v", err)
}
err = chanBucket.Put(remoteUnsignedLocalUpdatesKey, b2.Bytes())
if err != nil {
return fmt.Errorf("unable to restore chanbucket: %v", err)
}
return nil
})
if err != nil {
@@ -2065,6 +2102,39 @@ func (c *OpenChannel) UnsignedAckedUpdates() ([]LogUpdate, error) {
return updates, nil
}
// RemoteUnsignedLocalUpdates retrieves the persisted, unsigned local log
// updates that the remote still needs to sign for.
func (c *OpenChannel) RemoteUnsignedLocalUpdates() ([]LogUpdate, error) {
var updates []LogUpdate
err := kvdb.View(c.Db, func(tx kvdb.RTx) error {
chanBucket, err := fetchChanBucket(
tx, c.IdentityPub, &c.FundingOutpoint, c.ChainHash,
)
switch err {
case nil:
break
case ErrNoChanDBExists, ErrNoActiveChannels, ErrChannelNotFound:
return nil
default:
return err
}
updateBytes := chanBucket.Get(remoteUnsignedLocalUpdatesKey)
if updateBytes == nil {
return nil
}
r := bytes.NewReader(updateBytes)
updates, err = deserializeLogUpdates(r)
return err
})
if err != nil {
return nil, err
}
return updates, nil
}
// InsertNextRevocation inserts the _next_ commitment point (revocation) into
// the database, and also modifies the internal RemoteNextRevocation attribute
// to point to the passed key. This method is to be using during final channel
@@ -2101,8 +2171,12 @@ func (c *OpenChannel) InsertNextRevocation(revKey *btcec.PublicKey) error {
// this log can be consulted in order to reconstruct the state needed to
// rectify the situation. This method will add the current commitment for the
// remote party to the revocation log, and promote the current pending
// commitment to the current remote commitment.
func (c *OpenChannel) AdvanceCommitChainTail(fwdPkg *FwdPkg) error {
// commitment to the current remote commitment. The updates parameter is the
// set of local updates that the peer still needs to send us a signature for.
// We store this set of updates in case we go down.
func (c *OpenChannel) AdvanceCommitChainTail(fwdPkg *FwdPkg,
updates []LogUpdate) error {
c.Lock()
defer c.Unlock()
@@ -2226,6 +2300,20 @@ func (c *OpenChannel) AdvanceCommitChainTail(fwdPkg *FwdPkg) error {
return fmt.Errorf("unable to store under unsignedAckedUpdatesKey: %v", err)
}
// Persist the local updates the peer hasn't yet signed so they
// can be restored after restart.
var b2 bytes.Buffer
err = serializeLogUpdates(&b2, updates)
if err != nil {
return err
}
err = chanBucket.Put(remoteUnsignedLocalUpdatesKey, b2.Bytes())
if err != nil {
return fmt.Errorf("unable to restore remote unsigned "+
"local updates: %v", err)
}
newRemoteCommit = &newCommit.Commitment
return nil
@@ -2320,16 +2408,24 @@ func (c *OpenChannel) SetFwdFilter(height uint64, fwdFilter *PkgFilter) error {
})
}
// RemoveFwdPkg atomically removes a forwarding package specified by the remote
// commitment height.
// RemoveFwdPkgs atomically removes forwarding packages specified by the remote
// commitment heights. If one of the intermediate RemovePkg calls fails, then the
// later packages won't be removed.
//
// NOTE: This method should only be called on packages marked FwdStateCompleted.
func (c *OpenChannel) RemoveFwdPkg(height uint64) error {
func (c *OpenChannel) RemoveFwdPkgs(heights ...uint64) error {
c.Lock()
defer c.Unlock()
return kvdb.Update(c.Db, func(tx kvdb.RwTx) error {
return c.Packager.RemovePkg(tx, height)
for _, height := range heights {
err := c.Packager.RemovePkg(tx, height)
if err != nil {
return err
}
}
return nil
})
}
+12 -48
View File
@@ -2,10 +2,8 @@ package channeldb
import (
"bytes"
"io/ioutil"
"math/rand"
"net"
"os"
"reflect"
"runtime"
"testing"
@@ -86,40 +84,6 @@ var (
}
)
// makeTestDB creates a new instance of the ChannelDB for testing purposes. A
// callback which cleans up the created temporary directories is also returned
// and intended to be executed after the test completes.
func makeTestDB() (*DB, func(), error) {
// First, create a temporary directory to be used for the duration of
// this test.
tempDirName, err := ioutil.TempDir("", "channeldb")
if err != nil {
return nil, nil, err
}
// Next, create channeldb for the first time.
backend, backendCleanup, err := kvdb.GetTestBackend(tempDirName, "cdb")
if err != nil {
backendCleanup()
return nil, nil, err
}
cdb, err := CreateWithBackend(backend, OptionClock(testClock))
if err != nil {
backendCleanup()
os.RemoveAll(tempDirName)
return nil, nil, err
}
cleanUp := func() {
cdb.Close()
backendCleanup()
os.RemoveAll(tempDirName)
}
return cdb, cleanUp, nil
}
// testChannelParams is a struct which details the specifics of how a channel
// should be created.
type testChannelParams struct {
@@ -403,7 +367,7 @@ func createTestChannelState(t *testing.T, cdb *DB) *OpenChannel {
func TestOpenChannelPutGetDelete(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -552,7 +516,7 @@ func TestOptionalShutdown(t *testing.T) {
test := test
t.Run(test.name, func(t *testing.T) {
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -609,7 +573,7 @@ func assertCommitmentEqual(t *testing.T, a, b *ChannelCommitment) {
func TestChannelStateTransition(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -797,7 +761,7 @@ func TestChannelStateTransition(t *testing.T) {
fwdPkg := NewFwdPkg(channel.ShortChanID(), oldRemoteCommit.CommitHeight,
diskCommitDiff.LogUpdates, nil)
err = channel.AdvanceCommitChainTail(fwdPkg)
err = channel.AdvanceCommitChainTail(fwdPkg, nil)
if err != nil {
t.Fatalf("unable to append to revocation log: %v", err)
}
@@ -845,7 +809,7 @@ func TestChannelStateTransition(t *testing.T) {
fwdPkg = NewFwdPkg(channel.ShortChanID(), oldRemoteCommit.CommitHeight, nil, nil)
err = channel.AdvanceCommitChainTail(fwdPkg)
err = channel.AdvanceCommitChainTail(fwdPkg, nil)
if err != nil {
t.Fatalf("unable to append to revocation log: %v", err)
}
@@ -914,7 +878,7 @@ func TestChannelStateTransition(t *testing.T) {
func TestFetchPendingChannels(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -993,7 +957,7 @@ func TestFetchPendingChannels(t *testing.T) {
func TestFetchClosedChannels(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -1084,7 +1048,7 @@ func TestFetchWaitingCloseChannels(t *testing.T) {
// We'll start by creating two channels within our test database. One of
// them will have their funding transaction confirmed on-chain, while
// the other one will remain unconfirmed.
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -1199,7 +1163,7 @@ func TestFetchWaitingCloseChannels(t *testing.T) {
func TestRefreshShortChanID(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -1347,7 +1311,7 @@ func TestCloseInitiator(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v",
err)
@@ -1392,7 +1356,7 @@ func TestCloseInitiator(t *testing.T) {
// TestCloseChannelStatus tests setting of a channel status on the historical
// channel on channel close.
func TestCloseChannelStatus(t *testing.T) {
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v",
err)
@@ -1538,7 +1502,7 @@ func TestBalanceAtHeight(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v",
err)
+42
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/binary"
"fmt"
"io/ioutil"
"net"
"os"
@@ -163,6 +164,12 @@ var (
number: 17,
migration: mig.CreateTLB(closeSummaryBucket),
},
{
// Create a top level bucket which holds information
// about our peers.
number: 18,
migration: mig.CreateTLB(peersBucket),
},
}
// Big endian is the preferred byte order, due to cursor scans over
@@ -277,6 +284,7 @@ var topLevelBuckets = [][]byte{
invoiceBucket,
payAddrIndexBucket,
paymentsIndexBucket,
peersBucket,
nodeInfoBucket,
nodeBucket,
edgeBucket,
@@ -1260,3 +1268,37 @@ func (db *DB) FetchHistoricalChannel(outPoint *wire.OutPoint) (*OpenChannel, err
return channel, nil
}
// MakeTestDB creates a new instance of the ChannelDB for testing purposes.
// A callback which cleans up the created temporary directories is also
// returned and intended to be executed after the test completes.
func MakeTestDB(modifiers ...OptionModifier) (*DB, func(), error) {
// First, create a temporary directory to be used for the duration of
// this test.
tempDirName, err := ioutil.TempDir("", "channeldb")
if err != nil {
return nil, nil, err
}
// Next, create channeldb for the first time.
backend, backendCleanup, err := kvdb.GetTestBackend(tempDirName, "cdb")
if err != nil {
backendCleanup()
return nil, nil, err
}
cdb, err := CreateWithBackend(backend, modifiers...)
if err != nil {
backendCleanup()
os.RemoveAll(tempDirName)
return nil, nil, err
}
cleanUp := func() {
cdb.Close()
backendCleanup()
os.RemoveAll(tempDirName)
}
return cdb, cleanUp, nil
}
+8 -8
View File
@@ -115,7 +115,7 @@ func TestFetchClosedChannelForID(t *testing.T) {
const numChans = 101
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -186,7 +186,7 @@ func TestFetchClosedChannelForID(t *testing.T) {
func TestAddrsForNode(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -247,7 +247,7 @@ func TestAddrsForNode(t *testing.T) {
func TestFetchChannel(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -351,7 +351,7 @@ func genRandomChannelShell() (*ChannelShell, error) {
func TestRestoreChannelShells(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -398,7 +398,7 @@ func TestRestoreChannelShells(t *testing.T) {
if err != ErrNoRestoredChannelMutation {
t.Fatalf("able to mutate restored channel")
}
err = channel.AdvanceCommitChainTail(nil)
err = channel.AdvanceCommitChainTail(nil, nil)
if err != ErrNoRestoredChannelMutation {
t.Fatalf("able to mutate restored channel")
}
@@ -445,7 +445,7 @@ func TestRestoreChannelShells(t *testing.T) {
func TestAbandonChannel(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -618,7 +618,7 @@ func TestFetchChannels(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test "+
"database: %v", err)
@@ -687,7 +687,7 @@ func TestFetchChannels(t *testing.T) {
// TestFetchHistoricalChannel tests lookup of historical channels.
func TestFetchHistoricalChannel(t *testing.T) {
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
+85 -20
View File
@@ -6,6 +6,7 @@ import (
"sort"
"time"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/lnwire"
)
@@ -104,10 +105,9 @@ func decodeForwardingEvent(r io.Reader, f *ForwardingEvent) error {
func (f *ForwardingLog) AddForwardingEvents(events []ForwardingEvent) error {
// Before we create the database transaction, we'll ensure that the set
// of forwarding events are properly sorted according to their
// timestamp.
sort.Slice(events, func(i, j int) bool {
return events[i].Timestamp.Before(events[j].Timestamp)
})
// timestamp and that no duplicate timestamps exist to avoid collisions
// in the key we are going to store the events under.
makeUniqueTimestamps(events)
var timestamp [8]byte
@@ -124,22 +124,7 @@ func (f *ForwardingLog) AddForwardingEvents(events []ForwardingEvent) error {
// With the bucket obtained, we can now begin to write out the
// series of events.
for _, event := range events {
var eventBytes [forwardingEventSize]byte
eventBuf := bytes.NewBuffer(eventBytes[0:0:forwardingEventSize])
// First, we'll serialize this timestamp into our
// timestamp buffer.
byteOrder.PutUint64(
timestamp[:], uint64(event.Timestamp.UnixNano()),
)
// With the key encoded, we'll then encode the event
// into our buffer, then write it out to disk.
err := encodeForwardingEvent(eventBuf, &event)
if err != nil {
return err
}
err = logBucket.Put(timestamp[:], eventBuf.Bytes())
err := storeEvent(logBucket, event, timestamp[:])
if err != nil {
return err
}
@@ -149,6 +134,55 @@ func (f *ForwardingLog) AddForwardingEvents(events []ForwardingEvent) error {
})
}
// storeEvent tries to store a forwarding event into the given bucket by trying
// to avoid collisions. If a key for the event timestamp already exists in the
// database, the timestamp is incremented in nanosecond intervals until a "free"
// slot is found.
func storeEvent(bucket walletdb.ReadWriteBucket, event ForwardingEvent,
timestampScratchSpace []byte) error {
// First, we'll serialize this timestamp into our
// timestamp buffer.
byteOrder.PutUint64(
timestampScratchSpace, uint64(event.Timestamp.UnixNano()),
)
// Next we'll loop until we find a "free" slot in the bucket to store
// the event under. This should almost never happen unless we're running
// on a system that has a very bad system clock that doesn't properly
// resolve to nanosecond scale. We try up to 100 times (which would come
// to a maximum shift of 0.1 microsecond which is acceptable for most
// use cases). If we don't find a free slot, we just give up and let
// the collision happen. Something must be wrong with the data in that
// case, even on a very fast machine forwarding payments _will_ take a
// few microseconds at least so we should find a nanosecond slot
// somewhere.
const maxTries = 100
tries := 0
for tries < maxTries {
val := bucket.Get(timestampScratchSpace)
if val == nil {
break
}
// Collision, try the next nanosecond timestamp.
nextNano := event.Timestamp.UnixNano() + 1
event.Timestamp = time.Unix(0, nextNano)
byteOrder.PutUint64(timestampScratchSpace, uint64(nextNano))
tries++
}
// With the key encoded, we'll then encode the event
// into our buffer, then write it out to disk.
var eventBytes [forwardingEventSize]byte
eventBuf := bytes.NewBuffer(eventBytes[0:0:forwardingEventSize])
err := encodeForwardingEvent(eventBuf, &event)
if err != nil {
return err
}
return bucket.Put(timestampScratchSpace, eventBuf.Bytes())
}
// ForwardingEventQuery represents a query to the forwarding log payment
// circuit time series database. The query allows a caller to retrieve all
// records for a particular time slice, offset in that time slice, limiting the
@@ -272,3 +306,34 @@ func (f *ForwardingLog) Query(q ForwardingEventQuery) (ForwardingLogTimeSlice, e
return resp, nil
}
// makeUniqueTimestamps takes a slice of forwarding events, sorts it by the
// event timestamps and then makes sure there are no duplicates in the
// timestamps. If duplicates are found, some of the timestamps are increased on
// the nanosecond scale until only unique values remain. This is a fix to
// address the problem that in some environments (looking at you, Windows) the
// system clock has such a bad resolution that two serial invocations of
// time.Now() might return the same timestamp, even if some time has elapsed
// between the calls.
func makeUniqueTimestamps(events []ForwardingEvent) {
sort.Slice(events, func(i, j int) bool {
return events[i].Timestamp.Before(events[j].Timestamp)
})
// Now that we know the events are sorted by timestamp, we can go
// through the list and fix all duplicates until only unique values
// remain.
for outer := 0; outer < len(events)-1; outer++ {
current := events[outer].Timestamp.UnixNano()
next := events[outer+1].Timestamp.UnixNano()
// We initially sorted the slice. So if the current is now
// greater or equal to the next one, it's either because it's a
// duplicate or because we increased the current in the last
// iteration.
if current >= next {
next = current + 1
events[outer+1].Timestamp = time.Unix(0, next)
}
}
}
+126 -8
View File
@@ -4,11 +4,11 @@ import (
"math/rand"
"reflect"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/lightningnetwork/lnd/lnwire"
"time"
"github.com/stretchr/testify/assert"
)
// TestForwardingLogBasicStorageAndQuery tests that we're able to store and
@@ -19,11 +19,12 @@ func TestForwardingLogBasicStorageAndQuery(t *testing.T) {
// First, we'll set up a test database, and use that to instantiate the
// forwarding event log that we'll be using for the duration of the
// test.
db, cleanUp, err := makeTestDB()
defer cleanUp()
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
}
defer cleanUp()
log := ForwardingLog{
db: db,
}
@@ -91,11 +92,12 @@ func TestForwardingLogQueryOptions(t *testing.T) {
// First, we'll set up a test database, and use that to instantiate the
// forwarding event log that we'll be using for the duration of the
// test.
db, cleanUp, err := makeTestDB()
defer cleanUp()
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
}
defer cleanUp()
log := ForwardingLog{
db: db,
}
@@ -196,11 +198,12 @@ func TestForwardingLogQueryLimit(t *testing.T) {
// First, we'll set up a test database, and use that to instantiate the
// forwarding event log that we'll be using for the duration of the
// test.
db, cleanUp, err := makeTestDB()
defer cleanUp()
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
}
defer cleanUp()
log := ForwardingLog{
db: db,
}
@@ -263,3 +266,118 @@ func TestForwardingLogQueryLimit(t *testing.T) {
timeSlice.LastIndexOffset)
}
}
// TestForwardingLogMakeUniqueTimestamps makes sure the function that creates
// unique timestamps does it job correctly.
func TestForwardingLogMakeUniqueTimestamps(t *testing.T) {
t.Parallel()
// Create a list of events where some of the timestamps collide. We
// expect no existing timestamp to be overwritten, instead the "gaps"
// between them should be filled.
inputSlice := []ForwardingEvent{
{Timestamp: time.Unix(0, 1001)},
{Timestamp: time.Unix(0, 2001)},
{Timestamp: time.Unix(0, 1001)},
{Timestamp: time.Unix(0, 1002)},
{Timestamp: time.Unix(0, 1004)},
{Timestamp: time.Unix(0, 1004)},
{Timestamp: time.Unix(0, 1007)},
{Timestamp: time.Unix(0, 1001)},
}
expectedSlice := []ForwardingEvent{
{Timestamp: time.Unix(0, 1001)},
{Timestamp: time.Unix(0, 1002)},
{Timestamp: time.Unix(0, 1003)},
{Timestamp: time.Unix(0, 1004)},
{Timestamp: time.Unix(0, 1005)},
{Timestamp: time.Unix(0, 1006)},
{Timestamp: time.Unix(0, 1007)},
{Timestamp: time.Unix(0, 2001)},
}
makeUniqueTimestamps(inputSlice)
for idx, in := range inputSlice {
expect := expectedSlice[idx]
assert.Equal(
t, expect.Timestamp.UnixNano(), in.Timestamp.UnixNano(),
)
}
}
// TestForwardingLogStoreEvent makes sure forwarding events are stored without
// colliding on duplicate timestamps.
func TestForwardingLogStoreEvent(t *testing.T) {
t.Parallel()
// First, we'll set up a test database, and use that to instantiate the
// forwarding event log that we'll be using for the duration of the
// test.
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
}
defer cleanUp()
log := ForwardingLog{
db: db,
}
// We'll create 20 random events, with each event having a timestamp
// with just one nanosecond apart.
numEvents := 20
events := make([]ForwardingEvent, numEvents)
ts := time.Now().UnixNano()
for i := 0; i < numEvents; i++ {
events[i] = ForwardingEvent{
Timestamp: time.Unix(0, ts+int64(i)),
IncomingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
OutgoingChanID: lnwire.NewShortChanIDFromInt(uint64(rand.Int63())),
AmtIn: lnwire.MilliSatoshi(rand.Int63()),
AmtOut: lnwire.MilliSatoshi(rand.Int63()),
}
}
// Now that all of our events are constructed, we'll add them to the
// database in a batched manner.
if err := log.AddForwardingEvents(events); err != nil {
t.Fatalf("unable to add events: %v", err)
}
// Because timestamps are de-duplicated when adding them in a single
// batch before they even hit the DB, we add the same events again but
// in a new batch. They now have to be de-duplicated on the DB level.
if err := log.AddForwardingEvents(events); err != nil {
t.Fatalf("unable to add second batch of events: %v", err)
}
// With all of our events added, we should be able to query for all
// events with a range of just 40 nanoseconds (2 times 20 events, all
// spaced one nanosecond apart).
eventQuery := ForwardingEventQuery{
StartTime: time.Unix(0, ts),
EndTime: time.Unix(0, ts+int64(numEvents*2)),
IndexOffset: 0,
NumMaxEvents: uint32(numEvents * 3),
}
timeSlice, err := log.Query(eventQuery)
if err != nil {
t.Fatalf("unable to query for events: %v", err)
}
// We should get exactly 40 events back.
if len(timeSlice.ForwardingEvents) != numEvents*2 {
t.Fatalf("wrong number of events: expected %v, got %v",
numEvents*2, len(timeSlice.ForwardingEvents))
}
// The timestamps should be spaced out evenly and in order.
for i := 0; i < numEvents*2; i++ {
eventTs := timeSlice.ForwardingEvents[i].Timestamp.UnixNano()
if eventTs != ts+int64(i) {
t.Fatalf("unexpected timestamp of event %d: expected "+
"%d, got %d", i, ts+int64(i), eventTs)
}
}
}
+2 -2
View File
@@ -326,7 +326,7 @@ type SettleFailAcker interface {
type GlobalFwdPkgReader interface {
// LoadChannelFwdPkgs loads all known forwarding packages for the given
// channel.
LoadChannelFwdPkgs(tx kvdb.RwTx,
LoadChannelFwdPkgs(tx kvdb.RTx,
source lnwire.ShortChannelID) ([]*FwdPkg, error)
}
@@ -364,7 +364,7 @@ func (*SwitchPackager) AckSettleFails(tx kvdb.RwTx,
}
// LoadChannelFwdPkgs loads all forwarding packages for a particular channel.
func (*SwitchPackager) LoadChannelFwdPkgs(tx kvdb.RwTx,
func (*SwitchPackager) LoadChannelFwdPkgs(tx kvdb.RTx,
source lnwire.ShortChannelID) ([]*FwdPkg, error) {
return loadChannelFwdPkgs(tx, source)
+27 -27
View File
@@ -73,7 +73,7 @@ func createTestVertex(db *DB) (*LightningNode, error) {
func TestNodeInsertionAndDeletion(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -139,7 +139,7 @@ func TestNodeInsertionAndDeletion(t *testing.T) {
func TestPartialNode(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -201,7 +201,7 @@ func TestPartialNode(t *testing.T) {
func TestAliasLookup(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -255,7 +255,7 @@ func TestAliasLookup(t *testing.T) {
func TestSourceNode(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -296,7 +296,7 @@ func TestSourceNode(t *testing.T) {
func TestEdgeInsertionDeletion(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -431,7 +431,7 @@ func createEdge(height, txIndex uint32, txPosition uint16, outPointIndex uint32,
func TestDisconnectBlockAtHeight(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -718,7 +718,7 @@ func createChannelEdge(db *DB, node1, node2 *LightningNode) (*ChannelEdgeInfo,
func TestEdgeInfoUpdates(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -848,7 +848,7 @@ func newEdgePolicy(chanID uint64, op wire.OutPoint, db *DB,
func TestGraphTraversal(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1109,7 +1109,7 @@ func assertChanViewEqualChanPoints(t *testing.T, a []EdgePoint, b []*wire.OutPoi
func TestGraphPruning(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1317,7 +1317,7 @@ func TestGraphPruning(t *testing.T) {
func TestHighestChanID(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1394,7 +1394,7 @@ func TestHighestChanID(t *testing.T) {
func TestChanUpdatesInHorizon(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1570,7 +1570,7 @@ func TestChanUpdatesInHorizon(t *testing.T) {
func TestNodeUpdatesInHorizon(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1693,7 +1693,7 @@ func TestNodeUpdatesInHorizon(t *testing.T) {
func TestFilterKnownChanIDs(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1810,7 +1810,7 @@ func TestFilterKnownChanIDs(t *testing.T) {
func TestFilterChannelRange(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -1929,7 +1929,7 @@ func TestFilterChannelRange(t *testing.T) {
func TestFetchChanInfos(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2057,7 +2057,7 @@ func TestFetchChanInfos(t *testing.T) {
func TestIncompleteChannelPolicies(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2172,7 +2172,7 @@ func TestIncompleteChannelPolicies(t *testing.T) {
func TestChannelEdgePruningUpdateIndexDeletion(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2327,7 +2327,7 @@ func TestChannelEdgePruningUpdateIndexDeletion(t *testing.T) {
func TestPruneGraphNodes(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2411,7 +2411,7 @@ func TestPruneGraphNodes(t *testing.T) {
func TestAddChannelEdgeShellNodes(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2465,7 +2465,7 @@ func TestAddChannelEdgeShellNodes(t *testing.T) {
func TestNodePruningUpdateIndexDeletion(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2535,7 +2535,7 @@ func TestNodeIsPublic(t *testing.T) {
// We'll need to create a separate database and channel graph for each
// participant to replicate real-world scenarios (private edges being in
// some graphs but not others, etc.).
aliceDB, cleanUp, err := makeTestDB()
aliceDB, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2549,7 +2549,7 @@ func TestNodeIsPublic(t *testing.T) {
t.Fatalf("unable to set source node: %v", err)
}
bobDB, cleanUp, err := makeTestDB()
bobDB, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2563,7 +2563,7 @@ func TestNodeIsPublic(t *testing.T) {
t.Fatalf("unable to set source node: %v", err)
}
carolDB, cleanUp, err := makeTestDB()
carolDB, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2684,7 +2684,7 @@ func TestNodeIsPublic(t *testing.T) {
func TestDisabledChannelIDs(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -2782,7 +2782,7 @@ func TestDisabledChannelIDs(t *testing.T) {
func TestEdgePolicyMissingMaxHtcl(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
@@ -2962,7 +2962,7 @@ func TestGraphZombieIndex(t *testing.T) {
t.Parallel()
// We'll start by creating our test graph along with a test edge.
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to create test database: %v", err)
@@ -3151,7 +3151,7 @@ func TestLightningNodeSigVerification(t *testing.T) {
}
// Create a LightningNode from the same private key.
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
+139 -89
View File
@@ -136,7 +136,7 @@ func TestInvoiceWorkflow(t *testing.T) {
}
func testInvoiceWorkflow(t *testing.T, test invWorkflowTest) {
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
@@ -290,7 +290,7 @@ func testInvoiceWorkflow(t *testing.T, test invWorkflowTest) {
// TestAddDuplicatePayAddr asserts that the payment addresses of inserted
// invoices are unique.
func TestAddDuplicatePayAddr(t *testing.T) {
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
require.NoError(t, err)
@@ -317,7 +317,7 @@ func TestAddDuplicatePayAddr(t *testing.T) {
// addresses to be inserted if they are blank to support JIT legacy keysend
// invoices.
func TestAddDuplicateKeysendPayAddr(t *testing.T) {
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
require.NoError(t, err)
@@ -358,7 +358,7 @@ func TestAddDuplicateKeysendPayAddr(t *testing.T) {
// TestInvRefEquivocation asserts that retrieving or updating an invoice using
// an equivocating InvoiceRef results in ErrInvRefEquivocation.
func TestInvRefEquivocation(t *testing.T) {
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
require.NoError(t, err)
@@ -398,7 +398,7 @@ func TestInvRefEquivocation(t *testing.T) {
func TestInvoiceCancelSingleHtlc(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
@@ -472,7 +472,7 @@ func TestInvoiceCancelSingleHtlc(t *testing.T) {
func TestInvoiceAddTimeSeries(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB(OptionClock(testClock))
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
@@ -622,108 +622,65 @@ func TestInvoiceAddTimeSeries(t *testing.T) {
}
}
// Tests that FetchAllInvoicesWithPaymentHash returns all invoices with their
// corresponding payment hashes.
func TestFetchAllInvoicesWithPaymentHash(t *testing.T) {
// TestScanInvoices tests that ScanInvoices scans trough all stored invoices
// correctly.
func TestScanInvoices(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
}
// With an empty DB we expect to return no error and an empty list.
empty, err := db.FetchAllInvoicesWithPaymentHash(false)
if err != nil {
t.Fatalf("failed to call FetchAllInvoicesWithPaymentHash on empty DB: %v",
err)
var invoices map[lntypes.Hash]*Invoice
callCount := 0
resetCount := 0
// reset is used to reset/initialize results and is called once
// upon calling ScanInvoices and when the underlying transaction is
// retried.
reset := func() {
invoices = make(map[lntypes.Hash]*Invoice)
callCount = 0
resetCount++
}
if len(empty) != 0 {
t.Fatalf("expected empty list as a result, got: %v", empty)
scanFunc := func(paymentHash lntypes.Hash, invoice *Invoice) error {
invoices[paymentHash] = invoice
callCount++
return nil
}
states := []ContractState{
ContractOpen, ContractSettled, ContractCanceled, ContractAccepted,
}
// With an empty DB we expect to not scan any invoices.
require.NoError(t, db.ScanInvoices(scanFunc, reset))
require.Equal(t, 0, len(invoices))
require.Equal(t, 0, callCount)
require.Equal(t, 1, resetCount)
numInvoices := len(states) * 2
testPendingInvoices := make(map[lntypes.Hash]*Invoice)
testAllInvoices := make(map[lntypes.Hash]*Invoice)
numInvoices := 5
testInvoices := make(map[lntypes.Hash]*Invoice)
// Now populate the DB and check if we can get all invoices with their
// payment hashes as expected.
for i := 1; i <= numInvoices; i++ {
invoice, err := randInvoice(lnwire.MilliSatoshi(i))
if err != nil {
t.Fatalf("unable to create invoice: %v", err)
}
require.NoError(t, err)
// Set the contract state of the next invoice such that there's an equal
// number for all possbile states.
invoice.State = states[i%len(states)]
paymentHash := invoice.Terms.PaymentPreimage.Hash()
testInvoices[paymentHash] = invoice
if invoice.IsPending() {
testPendingInvoices[paymentHash] = invoice
}
testAllInvoices[paymentHash] = invoice
if _, err := db.AddInvoice(invoice, paymentHash); err != nil {
t.Fatalf("unable to add invoice: %v", err)
}
}
pendingInvoices, err := db.FetchAllInvoicesWithPaymentHash(true)
if err != nil {
t.Fatalf("can't fetch invoices with payment hash: %v", err)
}
if len(testPendingInvoices) != len(pendingInvoices) {
t.Fatalf("expected %v pending invoices, got: %v",
len(testPendingInvoices), len(pendingInvoices))
}
allInvoices, err := db.FetchAllInvoicesWithPaymentHash(false)
if err != nil {
t.Fatalf("can't fetch invoices with payment hash: %v", err)
}
if len(testAllInvoices) != len(allInvoices) {
t.Fatalf("expected %v invoices, got: %v",
len(testAllInvoices), len(allInvoices))
}
for i := range pendingInvoices {
expected, ok := testPendingInvoices[pendingInvoices[i].PaymentHash]
if !ok {
t.Fatalf("coulnd't find invoice with hash: %v",
pendingInvoices[i].PaymentHash)
}
// Zero out add index to not confuse require.Equal.
pendingInvoices[i].Invoice.AddIndex = 0
expected.AddIndex = 0
require.Equal(t, *expected, pendingInvoices[i].Invoice)
}
for i := range allInvoices {
expected, ok := testAllInvoices[allInvoices[i].PaymentHash]
if !ok {
t.Fatalf("coulnd't find invoice with hash: %v",
allInvoices[i].PaymentHash)
}
// Zero out add index to not confuse require.Equal.
allInvoices[i].Invoice.AddIndex = 0
expected.AddIndex = 0
require.Equal(t, *expected, allInvoices[i].Invoice)
_, err = db.AddInvoice(invoice, paymentHash)
require.NoError(t, err)
}
resetCount = 0
require.NoError(t, db.ScanInvoices(scanFunc, reset))
require.Equal(t, numInvoices, callCount)
require.Equal(t, testInvoices, invoices)
require.Equal(t, 1, resetCount)
}
// TestDuplicateSettleInvoice tests that if we add a new invoice and settle it
@@ -732,7 +689,7 @@ func TestFetchAllInvoicesWithPaymentHash(t *testing.T) {
func TestDuplicateSettleInvoice(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB(OptionClock(testClock))
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
@@ -797,7 +754,7 @@ func TestDuplicateSettleInvoice(t *testing.T) {
func TestQueryInvoices(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB(OptionClock(testClock))
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
@@ -1112,7 +1069,7 @@ func getUpdateInvoice(amt lnwire.MilliSatoshi) InvoiceUpdateCallback {
func TestCustomRecords(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatalf("unable to make test db: %v", err)
@@ -1194,3 +1151,96 @@ func TestInvoiceRef(t *testing.T) {
require.Equal(t, payHash, refByHashAndAddr.PayHash())
require.Equal(t, &payAddr, refByHashAndAddr.PayAddr())
}
// TestDeleteInvoices tests that deleting a list of invoices will succeed
// if all delete references are valid, or will fail otherwise.
func TestDeleteInvoices(t *testing.T) {
t.Parallel()
db, cleanup, err := MakeTestDB()
defer cleanup()
require.NoError(t, err, "unable to make test db")
// Add some invoices to the test db.
numInvoices := 3
invoicesToDelete := make([]InvoiceDeleteRef, numInvoices)
for i := 0; i < numInvoices; i++ {
invoice, err := randInvoice(lnwire.MilliSatoshi(i + 1))
require.NoError(t, err)
paymentHash := invoice.Terms.PaymentPreimage.Hash()
addIndex, err := db.AddInvoice(invoice, paymentHash)
require.NoError(t, err)
// Settle the second invoice.
if i == 1 {
invoice, err = db.UpdateInvoice(
InvoiceRefByHash(paymentHash),
getUpdateInvoice(invoice.Terms.Value),
)
require.NoError(t, err, "unable to settle invoice")
}
// store the delete ref for later.
invoicesToDelete[i] = InvoiceDeleteRef{
PayHash: paymentHash,
PayAddr: &invoice.Terms.PaymentAddr,
AddIndex: addIndex,
SettleIndex: invoice.SettleIndex,
}
}
// assertInvoiceCount asserts that the number of invoices equals
// to the passed count.
assertInvoiceCount := func(count int) {
// Query to collect all invoices.
query := InvoiceQuery{
IndexOffset: 0,
NumMaxInvoices: math.MaxUint64,
}
// Check that we really have 3 invoices.
response, err := db.QueryInvoices(query)
require.NoError(t, err)
require.Equal(t, count, len(response.Invoices))
}
// XOR one byte of one of the references' hash and attempt to delete.
invoicesToDelete[0].PayHash[2] ^= 3
require.Error(t, db.DeleteInvoice(invoicesToDelete))
assertInvoiceCount(3)
// Restore the hash.
invoicesToDelete[0].PayHash[2] ^= 3
// XOR one byte of one of the references' payment address and attempt
// to delete.
invoicesToDelete[1].PayAddr[5] ^= 7
require.Error(t, db.DeleteInvoice(invoicesToDelete))
assertInvoiceCount(3)
// Restore the payment address.
invoicesToDelete[1].PayAddr[5] ^= 7
// XOR the second invoice's payment settle index as it is settled, and
// attempt to delete.
invoicesToDelete[1].SettleIndex ^= 11
require.Error(t, db.DeleteInvoice(invoicesToDelete))
assertInvoiceCount(3)
// Restore the settle index.
invoicesToDelete[1].SettleIndex ^= 11
// XOR the add index for one of the references and attempt to delete.
invoicesToDelete[2].AddIndex ^= 13
require.Error(t, db.DeleteInvoice(invoicesToDelete))
assertInvoiceCount(3)
// Restore the add index.
invoicesToDelete[2].AddIndex ^= 13
// Delete should succeed with all the valid references.
require.NoError(t, db.DeleteInvoice(invoicesToDelete))
assertInvoiceCount(0)
}
+147 -37
View File
@@ -723,28 +723,21 @@ func fetchInvoiceNumByRef(invoiceIndex, payAddrIndex kvdb.RBucket,
}
}
// InvoiceWithPaymentHash is used to store an invoice and its corresponding
// payment hash. This struct is only used to store results of
// ChannelDB.FetchAllInvoicesWithPaymentHash() call.
type InvoiceWithPaymentHash struct {
// Invoice holds the invoice as selected from the invoices bucket.
Invoice Invoice
// ScanInvoices scans trough all invoices and calls the passed scanFunc for
// for each invoice with its respective payment hash. Additionally a reset()
// closure is passed which is used to reset/initialize partial results and also
// to signal if the kvdb.View transaction has been retried.
func (d *DB) ScanInvoices(
scanFunc func(lntypes.Hash, *Invoice) error, reset func()) error {
// PaymentHash is the payment hash for the Invoice.
PaymentHash lntypes.Hash
}
return kvdb.View(d, func(tx kvdb.RTx) error {
// Reset partial results. As transaction commit success is not
// guaranteed when using etcd, we need to be prepared to redo
// the whole view transaction. In order to be able to do that
// we need a way to reset existing results. This is also done
// upon first run for initialization.
reset()
// FetchAllInvoicesWithPaymentHash returns all invoices and their payment hashes
// currently stored within the database. If the pendingOnly param is true, then
// only open or accepted invoices and their payment hashes will be returned,
// skipping all invoices that are fully settled or canceled. Note that the
// returned array is not ordered by add index.
func (d *DB) FetchAllInvoicesWithPaymentHash(pendingOnly bool) (
[]InvoiceWithPaymentHash, error) {
var result []InvoiceWithPaymentHash
err := kvdb.View(d, func(tx kvdb.RTx) error {
invoices := tx.ReadBucket(invoiceBucket)
if invoices == nil {
return ErrNoInvoicesCreated
@@ -775,26 +768,12 @@ func (d *DB) FetchAllInvoicesWithPaymentHash(pendingOnly bool) (
return err
}
if pendingOnly && !invoice.IsPending() {
return nil
}
var paymentHash lntypes.Hash
copy(paymentHash[:], k)
invoiceWithPaymentHash := InvoiceWithPaymentHash{
Invoice: invoice,
}
copy(invoiceWithPaymentHash.PaymentHash[:], k)
result = append(result, invoiceWithPaymentHash)
return nil
return scanFunc(paymentHash, &invoice)
})
})
if err != nil {
return nil, err
}
return result, nil
}
// InvoiceQuery represents a query to the invoice database. The query allows a
@@ -1761,3 +1740,134 @@ func setSettleMetaFields(settleIndex kvdb.RwBucket, invoiceNum []byte,
return nil
}
// InvoiceDeleteRef holds a refererence to an invoice to be deleted.
type InvoiceDeleteRef struct {
// PayHash is the payment hash of the target invoice. All invoices are
// currently indexed by payment hash.
PayHash lntypes.Hash
// PayAddr is the payment addr of the target invoice. Newer invoices
// (0.11 and up) are indexed by payment address in addition to payment
// hash, but pre 0.8 invoices do not have one at all.
PayAddr *[32]byte
// AddIndex is the add index of the invoice.
AddIndex uint64
// SettleIndex is the settle index of the invoice.
SettleIndex uint64
}
// DeleteInvoice attempts to delete the passed invoices from the database in
// one transaction. The passed delete references hold all keys required to
// delete the invoices without also needing to deserialze them.
func (d *DB) DeleteInvoice(invoicesToDelete []InvoiceDeleteRef) error {
err := kvdb.Update(d, func(tx kvdb.RwTx) error {
invoices := tx.ReadWriteBucket(invoiceBucket)
if invoices == nil {
return ErrNoInvoicesCreated
}
invoiceIndex := invoices.NestedReadWriteBucket(
invoiceIndexBucket,
)
if invoiceIndex == nil {
return ErrNoInvoicesCreated
}
invoiceAddIndex := invoices.NestedReadWriteBucket(
addIndexBucket,
)
if invoiceAddIndex == nil {
return ErrNoInvoicesCreated
}
// settleIndex can be nil, as the bucket is created lazily
// when the first invoice is settled.
settleIndex := invoices.NestedReadWriteBucket(settleIndexBucket)
payAddrIndex := tx.ReadWriteBucket(payAddrIndexBucket)
for _, ref := range invoicesToDelete {
// Fetch the invoice key for using it to check for
// consistency and also to delete from the invoice index.
invoiceKey := invoiceIndex.Get(ref.PayHash[:])
if invoiceKey == nil {
return ErrInvoiceNotFound
}
err := invoiceIndex.Delete(ref.PayHash[:])
if err != nil {
return err
}
// Delete payment address index reference if there's a
// valid payment address passed.
if ref.PayAddr != nil {
// To ensure consistency check that the already
// fetched invoice key matches the one in the
// payment address index.
key := payAddrIndex.Get(ref.PayAddr[:])
if !bytes.Equal(key, invoiceKey) {
return fmt.Errorf("unknown invoice")
}
// Delete from the payment address index.
err := payAddrIndex.Delete(ref.PayAddr[:])
if err != nil {
return err
}
}
var addIndexKey [8]byte
byteOrder.PutUint64(addIndexKey[:], ref.AddIndex)
// To ensure consistency check that the key stored in
// the add index also matches the previously fetched
// invoice key.
key := invoiceAddIndex.Get(addIndexKey[:])
if !bytes.Equal(key, invoiceKey) {
return fmt.Errorf("unknown invoice")
}
// Remove from the add index.
err = invoiceAddIndex.Delete(addIndexKey[:])
if err != nil {
return err
}
// Remove from the settle index if available and
// if the invoice is settled.
if settleIndex != nil && ref.SettleIndex > 0 {
var settleIndexKey [8]byte
byteOrder.PutUint64(
settleIndexKey[:], ref.SettleIndex,
)
// To ensure consistency check that the already
// fetched invoice key matches the one in the
// settle index
key := settleIndex.Get(settleIndexKey[:])
if !bytes.Equal(key, invoiceKey) {
return fmt.Errorf("unknown invoice")
}
err = settleIndex.Delete(settleIndexKey[:])
if err != nil {
return err
}
}
// Finally remove the serialized invoice from the
// invoice bucket.
err = invoices.Delete(invoiceKey)
if err != nil {
return err
}
}
return nil
})
return err
}
+1 -1
View File
@@ -12,7 +12,7 @@ const EtcdBackendName = "etcd"
// BoltConfig holds bolt configuration.
type BoltConfig struct {
NoFreeListSync bool `long:"nofreelistsync" description:"If true, prevents the database from syncing its freelist to disk"`
SyncFreelist bool `long:"nofreelistsync" description:"Whether the databases used within lnd should sync their freelist to disk. This is disabled by default resulting in improved memory performance during operation, but with an increase in startup time."`
}
// EtcdConfig holds etcd configuration.
+44 -31
View File
@@ -11,9 +11,9 @@ const (
)
var (
bucketPrefix = []byte("b")
valuePrefix = []byte("v")
sequencePrefix = []byte("$")
valuePostfix = []byte{0x00}
bucketPostfix = []byte{0xFF}
sequencePrefix = []byte("$seq$")
)
// makeBucketID returns a deterministic key for the passed byte slice.
@@ -28,52 +28,65 @@ func isValidBucketID(s []byte) bool {
return len(s) == bucketIDLength
}
// makeKey concatenates prefix, parent and key into one byte slice.
// The prefix indicates the use of this key (whether bucket, value or sequence),
// while parentID refers to the parent bucket.
func makeKey(prefix, parent, key []byte) []byte {
keyBuf := make([]byte, len(prefix)+len(parent)+len(key))
copy(keyBuf, prefix)
copy(keyBuf[len(prefix):], parent)
copy(keyBuf[len(prefix)+len(parent):], key)
// makeKey concatenates parent, key and postfix into one byte slice.
// The postfix indicates the use of this key (whether bucket or value), while
// parent refers to the parent bucket.
func makeKey(parent, key, postfix []byte) []byte {
keyBuf := make([]byte, len(parent)+len(key)+len(postfix))
copy(keyBuf, parent)
copy(keyBuf[len(parent):], key)
copy(keyBuf[len(parent)+len(key):], postfix)
return keyBuf
}
// makePrefix concatenates prefix with parent into one byte slice.
func makePrefix(prefix []byte, parent []byte) []byte {
prefixBuf := make([]byte, len(prefix)+len(parent))
copy(prefixBuf, prefix)
copy(prefixBuf[len(prefix):], parent)
return prefixBuf
}
// makeBucketKey returns a bucket key from the passed parent bucket id and
// the key.
func makeBucketKey(parent []byte, key []byte) []byte {
return makeKey(bucketPrefix, parent, key)
return makeKey(parent, key, bucketPostfix)
}
// makeValueKey returns a value key from the passed parent bucket id and
// the key.
func makeValueKey(parent []byte, key []byte) []byte {
return makeKey(valuePrefix, parent, key)
return makeKey(parent, key, valuePostfix)
}
// makeSequenceKey returns a sequence key of the passed parent bucket id.
func makeSequenceKey(parent []byte) []byte {
return makeKey(sequencePrefix, parent, nil)
keyBuf := make([]byte, len(sequencePrefix)+len(parent))
copy(keyBuf, sequencePrefix)
copy(keyBuf[len(sequencePrefix):], parent)
return keyBuf
}
// makeBucketPrefix returns the bucket prefix of the passed parent bucket id.
// This prefix is used for all sub buckets.
func makeBucketPrefix(parent []byte) []byte {
return makePrefix(bucketPrefix, parent)
// isBucketKey returns true if the passed key is a bucket key, meaning it
// keys a bucket name.
func isBucketKey(key string) bool {
if len(key) < bucketIDLength+1 {
return false
}
return key[len(key)-1] == bucketPostfix[0]
}
// makeValuePrefix returns the value prefix of the passed parent bucket id.
// This prefix is used for all key/values in the bucket.
func makeValuePrefix(parent []byte) []byte {
return makePrefix(valuePrefix, parent)
// getKey chops out the key from the raw key (by removing the bucket id
// prefixing the key and the postfix indicating whether it is a bucket or
// a value key)
func getKey(rawKey string) []byte {
return []byte(rawKey[bucketIDLength : len(rawKey)-1])
}
// getKeyVal chops out the key from the raw key (by removing the bucket id
// prefixing the key and the postfix indicating whether it is a bucket or
// a value key) and also returns the appropriate value for the key, which is
// nil in case of buckets (or the set value otherwise).
func getKeyVal(kv *KV) ([]byte, []byte) {
var val []byte
if !isBucketKey(kv.key) {
val = []byte(kv.val)
}
return getKey(kv.key), val
}
+150
View File
@@ -0,0 +1,150 @@
// +build kvdb_etcd
package etcd
import (
"context"
"sync"
)
// commitQueueSize is the maximum number of commits we let to queue up. All
// remaining commits will block on commitQueue.Add().
const commitQueueSize = 100
// commitQueue is a simple execution queue to manage conflicts for transactions
// and thereby reduce the number of times conflicting transactions need to be
// retried. When a new transaction is added to the queue, we first upgrade the
// read/write counts in the queue's own accounting to decide whether the new
// transaction has any conflicting dependencies. If the transaction does not
// conflict with any other, then it is comitted immediately, otherwise it'll be
// queued up for later exection.
// The algorithm is described in: http://www.cs.umd.edu/~abadi/papers/vll-vldb13.pdf
type commitQueue struct {
ctx context.Context
mx sync.Mutex
readerMap map[string]int
writerMap map[string]int
commitMutex sync.RWMutex
queue chan (func())
wg sync.WaitGroup
}
// NewCommitQueue creates a new commit queue, with the passed abort context.
func NewCommitQueue(ctx context.Context) *commitQueue {
q := &commitQueue{
ctx: ctx,
readerMap: make(map[string]int),
writerMap: make(map[string]int),
queue: make(chan func(), commitQueueSize),
}
// Start the queue consumer loop.
q.wg.Add(1)
go q.mainLoop()
return q
}
// Wait waits for the queue to stop (after the queue context has been canceled).
func (c *commitQueue) Wait() {
c.wg.Wait()
}
// Add increases lock counts and queues up tx commit closure for execution.
// Transactions that don't have any conflicts are executed immediately by
// "downgrading" the count mutex to allow concurrency.
func (c *commitQueue) Add(commitLoop func(), rset readSet, wset writeSet) {
c.mx.Lock()
blocked := false
// Mark as blocked if there's any writer changing any of the keys in
// the read set. Do not increment the reader counts yet as we'll need to
// use the original reader counts when scanning through the write set.
for key := range rset {
if c.writerMap[key] > 0 {
blocked = true
break
}
}
// Mark as blocked if there's any writer or reader for any of the keys
// in the write set.
for key := range wset {
blocked = blocked || c.readerMap[key] > 0 || c.writerMap[key] > 0
// Increment the writer count.
c.writerMap[key] += 1
}
// Finally we can increment the reader counts for keys in the read set.
for key := range rset {
c.readerMap[key] += 1
}
if blocked {
// Add the transaction to the queue if conflicts with an already
// queued one.
c.mx.Unlock()
select {
case c.queue <- commitLoop:
case <-c.ctx.Done():
}
} else {
// To make sure we don't add a new tx to the queue that depends
// on this "unblocked" tx, grab the commitMutex before lifting
// the mutex guarding the lock maps.
c.commitMutex.RLock()
c.mx.Unlock()
// At this point we're safe to execute the "unblocked" tx, as
// we cannot execute blocked tx that may have been read from the
// queue until the commitMutex is held.
commitLoop()
c.commitMutex.RUnlock()
}
}
// Done decreases lock counts of the keys in the read/write sets.
func (c *commitQueue) Done(rset readSet, wset writeSet) {
c.mx.Lock()
defer c.mx.Unlock()
for key := range rset {
c.readerMap[key] -= 1
if c.readerMap[key] == 0 {
delete(c.readerMap, key)
}
}
for key := range wset {
c.writerMap[key] -= 1
if c.writerMap[key] == 0 {
delete(c.writerMap, key)
}
}
}
// mainLoop executes queued transaction commits for transactions that have
// dependencies. The queue ensures that the top element doesn't conflict with
// any other transactions and therefore can be executed freely.
func (c *commitQueue) mainLoop() {
defer c.wg.Done()
for {
select {
case top := <-c.queue:
// Execute the next blocked transaction. As it is
// the top element in the queue it means that it doesn't
// depend on any other transactions anymore.
c.commitMutex.Lock()
top()
c.commitMutex.Unlock()
case <-c.ctx.Done():
return
}
}
}
+115
View File
@@ -0,0 +1,115 @@
// +build kvdb_etcd
package etcd
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
)
// TestCommitQueue tests that non-conflicting transactions commit concurrently,
// while conflicting transactions are queued up.
func TestCommitQueue(t *testing.T) {
// The duration of each commit.
const commitDuration = time.Millisecond * 500
const numCommits = 4
var wg sync.WaitGroup
commits := make([]string, numCommits)
idx := int32(-1)
commit := func(tag string, sleep bool) func() {
return func() {
defer wg.Done()
// Update our log of commit order. Avoid blocking
// by preallocating the commit log and increasing
// the log index atomically.
i := atomic.AddInt32(&idx, 1)
commits[i] = tag
if sleep {
time.Sleep(commitDuration)
}
}
}
// Helper function to create a read set from the passed keys.
makeReadSet := func(keys []string) readSet {
rs := make(map[string]stmGet)
for _, key := range keys {
rs[key] = stmGet{}
}
return rs
}
// Helper function to create a write set from the passed keys.
makeWriteSet := func(keys []string) writeSet {
ws := make(map[string]stmPut)
for _, key := range keys {
ws[key] = stmPut{}
}
return ws
}
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
q := NewCommitQueue(ctx)
defer q.Wait()
defer cancel()
wg.Add(numCommits)
t1 := time.Now()
// Tx1: reads: key1, key2, writes: key3, conflict: none
q.Add(
commit("free", true),
makeReadSet([]string{"key1", "key2"}),
makeWriteSet([]string{"key3"}),
)
// Tx2: reads: key1, key2, writes: key3, conflict: Tx1
q.Add(
commit("blocked1", false),
makeReadSet([]string{"key1", "key2"}),
makeWriteSet([]string{"key3"}),
)
// Tx3: reads: key1, writes: key4, conflict: none
q.Add(
commit("free", true),
makeReadSet([]string{"key1", "key2"}),
makeWriteSet([]string{"key4"}),
)
// Tx4: reads: key2, writes: key4 conflict: Tx3
q.Add(
commit("blocked2", false),
makeReadSet([]string{"key2"}),
makeWriteSet([]string{"key4"}),
)
// Wait for all commits.
wg.Wait()
t2 := time.Now()
// Expected total execution time: delta.
// 2 * commitDuration <= delta < 3 * commitDuration
delta := t2.Sub(t1)
require.LessOrEqual(t, int64(commitDuration*2), int64(delta))
require.Greater(t, int64(commitDuration*3), int64(delta))
// Expect that the non-conflicting "free" transactions are executed
// before the blocking ones, and the blocking ones are executed in
// the order of addition.
require.Equal(t,
[]string{"free", "free", "blocked1", "blocked2"},
commits,
)
}
+24 -18
View File
@@ -16,8 +16,8 @@ import (
)
const (
// etcdConnectionTimeout is the timeout until successful connection to the
// etcd instance.
// etcdConnectionTimeout is the timeout until successful connection to
// the etcd instance.
etcdConnectionTimeout = 10 * time.Second
// etcdLongTimeout is a timeout for longer taking etcd operatons.
@@ -34,7 +34,8 @@ type callerStats struct {
func (s callerStats) String() string {
return fmt.Sprintf("count: %d, retries: %d, rset: %d, wset: %d",
s.count, s.commitStats.Retries, s.commitStats.Rset, s.commitStats.Wset)
s.count, s.commitStats.Retries, s.commitStats.Rset,
s.commitStats.Wset)
}
// commitStatsCollector collects commit stats for commits succeeding
@@ -117,6 +118,7 @@ type db struct {
config BackendConfig
cli *clientv3.Client
commitStatsCollector *commitStatsCollector
txQueue *commitQueue
}
// Enforce db implements the walletdb.DB interface.
@@ -174,12 +176,13 @@ func newEtcdBackend(config BackendConfig) (*db, error) {
}
cli, err := clientv3.New(clientv3.Config{
Context: config.Ctx,
Endpoints: []string{config.Host},
DialTimeout: etcdConnectionTimeout,
Username: config.User,
Password: config.Pass,
TLS: tlsConfig,
Context: config.Ctx,
Endpoints: []string{config.Host},
DialTimeout: etcdConnectionTimeout,
Username: config.User,
Password: config.Pass,
TLS: tlsConfig,
MaxCallSendMsgSize: 16384*1024 - 1,
})
if err != nil {
@@ -187,8 +190,9 @@ func newEtcdBackend(config BackendConfig) (*db, error) {
}
backend := &db{
cli: cli,
config: config,
cli: cli,
config: config,
txQueue: NewCommitQueue(config.Ctx),
}
if config.CollectCommitStats {
@@ -200,7 +204,9 @@ func newEtcdBackend(config BackendConfig) (*db, error) {
// getSTMOptions creats all STM options based on the backend config.
func (db *db) getSTMOptions() []STMOptionFunc {
opts := []STMOptionFunc{WithAbortContext(db.config.Ctx)}
opts := []STMOptionFunc{
WithAbortContext(db.config.Ctx),
}
if db.config.CollectCommitStats {
opts = append(opts,
@@ -220,7 +226,7 @@ func (db *db) View(f func(tx walletdb.ReadTx) error) error {
return f(newReadWriteTx(stm, db.config.Prefix))
}
return RunSTM(db.cli, apply, db.getSTMOptions()...)
return RunSTM(db.cli, apply, db.txQueue, db.getSTMOptions()...)
}
// Update opens a database read/write transaction and executes the function f
@@ -234,7 +240,7 @@ func (db *db) Update(f func(tx walletdb.ReadWriteTx) error) error {
return f(newReadWriteTx(stm, db.config.Prefix))
}
return RunSTM(db.cli, apply, db.getSTMOptions()...)
return RunSTM(db.cli, apply, db.txQueue, db.getSTMOptions()...)
}
// PrintStats returns all collected stats pretty printed into a string.
@@ -246,18 +252,18 @@ func (db *db) PrintStats() string {
return ""
}
// BeginReadTx opens a database read transaction.
// BeginReadWriteTx opens a database read+write transaction.
func (db *db) BeginReadWriteTx() (walletdb.ReadWriteTx, error) {
return newReadWriteTx(
NewSTM(db.cli, db.getSTMOptions()...),
NewSTM(db.cli, db.txQueue, db.getSTMOptions()...),
db.config.Prefix,
), nil
}
// BeginReadWriteTx opens a database read+write transaction.
// BeginReadTx opens a database read transaction.
func (db *db) BeginReadTx() (walletdb.ReadTx, error) {
return newReadWriteTx(
NewSTM(db.cli, db.getSTMOptions()...),
NewSTM(db.cli, db.txQueue, db.getSTMOptions()...),
db.config.Prefix,
), nil
}
+13 -13
View File
@@ -8,7 +8,7 @@ import (
"testing"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCopy(t *testing.T) {
@@ -18,30 +18,30 @@ func TestCopy(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
// "apple"
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.NoError(t, err)
assert.NotNil(t, apple)
require.NoError(t, err)
require.NotNil(t, apple)
assert.NoError(t, apple.Put([]byte("key"), []byte("val")))
require.NoError(t, apple.Put([]byte("key"), []byte("val")))
return nil
})
// Expect non-zero copy.
var buf bytes.Buffer
assert.NoError(t, db.Copy(&buf))
assert.Greater(t, buf.Len(), 0)
assert.Nil(t, err)
require.NoError(t, db.Copy(&buf))
require.Greater(t, buf.Len(), 0)
require.Nil(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
vkey("key", "apple"): "val",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
func TestAbortContext(t *testing.T) {
@@ -57,19 +57,19 @@ func TestAbortContext(t *testing.T) {
// Pass abort context and abort right away.
db, err := newEtcdBackend(config)
assert.NoError(t, err)
require.NoError(t, err)
cancel()
// Expect that the update will fail.
err = db.Update(func(tx walletdb.ReadWriteTx) error {
_, err := tx.CreateTopLevelBucket([]byte("bucket"))
assert.NoError(t, err)
require.Error(t, err, "context canceled")
return nil
})
assert.Error(t, err, "context canceled")
require.Error(t, err, "context canceled")
// No changes in the DB.
assert.Equal(t, map[string]string{}, f.Dump())
require.Equal(t, map[string]string{}, f.Dump())
}
+9 -9
View File
@@ -6,25 +6,25 @@ import (
"testing"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestOpenCreateFailure(t *testing.T) {
t.Parallel()
db, err := walletdb.Open(dbType)
assert.Error(t, err)
assert.Nil(t, db)
require.Error(t, err)
require.Nil(t, db)
db, err = walletdb.Open(dbType, "wrong")
assert.Error(t, err)
assert.Nil(t, db)
require.Error(t, err)
require.Nil(t, db)
db, err = walletdb.Create(dbType)
assert.Error(t, err)
assert.Nil(t, db)
require.Error(t, err)
require.Nil(t, db)
db, err = walletdb.Create(dbType, "wrong")
assert.Error(t, err)
assert.Nil(t, db)
require.Error(t, err)
require.Nil(t, db)
}
+6 -2
View File
@@ -42,7 +42,8 @@ func NewEmbeddedEtcdInstance(path string) (*BackendConfig, func(), error) {
cfg.Dir = path
// To ensure that we can submit large transactions.
cfg.MaxTxnOps = 1000
cfg.MaxTxnOps = 8192
cfg.MaxRequestBytes = 16384 * 1024
// Listen on random free ports.
clientURL := fmt.Sprintf("127.0.0.1:%d", getFreePort())
@@ -63,8 +64,10 @@ func NewEmbeddedEtcdInstance(path string) (*BackendConfig, func(), error) {
fmt.Errorf("etcd failed to start after: %v", readyTimeout)
}
ctx, cancel := context.WithCancel(context.Background())
connConfig := &BackendConfig{
Ctx: context.Background(),
Ctx: ctx,
Host: "http://" + peerURL,
User: "user",
Pass: "pass",
@@ -72,6 +75,7 @@ func NewEmbeddedEtcdInstance(path string) (*BackendConfig, func(), error) {
}
return connConfig, func() {
cancel()
etcd.Close()
}, nil
}
+54 -65
View File
@@ -3,7 +3,6 @@
package etcd
import (
"bytes"
"strconv"
"github.com/btcsuite/btcwallet/walletdb"
@@ -24,11 +23,6 @@ type readWriteBucket struct {
// newReadWriteBucket creates a new rw bucket with the passed transaction
// and bucket id.
func newReadWriteBucket(tx *readWriteTx, key, id []byte) *readWriteBucket {
if !bytes.Equal(id, tx.rootBucketID[:]) {
// Add the bucket key/value to the lock set.
tx.lock(string(key), string(id))
}
return &readWriteBucket{
id: id,
tx: tx,
@@ -46,44 +40,23 @@ func (b *readWriteBucket) NestedReadBucket(key []byte) walletdb.ReadBucket {
// is nil, but it does not include the key/value pairs within those
// nested buckets.
func (b *readWriteBucket) ForEach(cb func(k, v []byte) error) error {
prefix := makeValuePrefix(b.id)
prefixLen := len(prefix)
prefix := string(b.id)
// Get the first matching key that is in the bucket.
kv, err := b.tx.stm.First(string(prefix))
kv, err := b.tx.stm.First(prefix)
if err != nil {
return err
}
for kv != nil {
if err := cb([]byte(kv.key[prefixLen:]), []byte(kv.val)); err != nil {
key, val := getKeyVal(kv)
if err := cb(key, val); err != nil {
return err
}
// Step to the next key.
kv, err = b.tx.stm.Next(string(prefix), kv.key)
if err != nil {
return err
}
}
// Make a bucket prefix. This prefixes all sub buckets.
prefix = makeBucketPrefix(b.id)
prefixLen = len(prefix)
// Get the first bucket.
kv, err = b.tx.stm.First(string(prefix))
if err != nil {
return err
}
for kv != nil {
if err := cb([]byte(kv.key[prefixLen:]), nil); err != nil {
return err
}
// Step to the next bucket.
kv, err = b.tx.stm.Next(string(prefix), kv.key)
kv, err = b.tx.stm.Next(prefix, kv.key)
if err != nil {
return err
}
@@ -143,6 +116,20 @@ func (b *readWriteBucket) NestedReadWriteBucket(key []byte) walletdb.ReadWriteBu
return newReadWriteBucket(b.tx, bucketKey, bucketVal)
}
// assertNoValue checks if the value for the passed key exists.
func (b *readWriteBucket) assertNoValue(key []byte) error {
val, err := b.tx.stm.Get(string(makeValueKey(b.id, key)))
if err != nil {
return err
}
if val != nil {
return walletdb.ErrIncompatibleValue
}
return nil
}
// CreateBucket creates and returns a new nested bucket with the given
// key. Returns ErrBucketExists if the bucket already exists,
// ErrBucketNameRequired if the key is empty, or ErrIncompatibleValue
@@ -168,11 +155,15 @@ func (b *readWriteBucket) CreateBucket(key []byte) (
return nil, walletdb.ErrBucketExists
}
if err := b.assertNoValue(key); err != nil {
return nil, err
}
// Create a deterministic bucket id from the bucket key.
newID := makeBucketID(bucketKey)
// Create the bucket.
b.tx.put(string(bucketKey), string(newID[:]))
b.tx.stm.Put(string(bucketKey), string(newID[:]))
return newReadWriteBucket(b.tx, bucketKey, newID[:]), nil
}
@@ -198,8 +189,12 @@ func (b *readWriteBucket) CreateBucketIfNotExists(key []byte) (
}
if !isValidBucketID(bucketVal) {
if err := b.assertNoValue(key); err != nil {
return nil, err
}
newID := makeBucketID(bucketKey)
b.tx.put(string(bucketKey), string(newID[:]))
b.tx.stm.Put(string(bucketKey), string(newID[:]))
return newReadWriteBucket(b.tx, bucketKey, newID[:]), nil
}
@@ -241,46 +236,31 @@ func (b *readWriteBucket) DeleteNestedBucket(key []byte) error {
id := queue[0]
queue = queue[1:]
// Delete values in the current bucket
valuePrefix := string(makeValuePrefix(id))
kv, err := b.tx.stm.First(valuePrefix)
kv, err := b.tx.stm.First(string(id))
if err != nil {
return err
}
for kv != nil {
b.tx.del(kv.key)
b.tx.stm.Del(kv.key)
kv, err = b.tx.stm.Next(valuePrefix, kv.key)
if isBucketKey(kv.key) {
queue = append(queue, []byte(kv.val))
}
kv, err = b.tx.stm.Next(string(id), kv.key)
if err != nil {
return err
}
}
// Iterate sub buckets
bucketPrefix := string(makeBucketPrefix(id))
kv, err = b.tx.stm.First(bucketPrefix)
if err != nil {
return err
}
for kv != nil {
// Delete sub bucket key.
b.tx.del(kv.key)
// Queue it for traversal.
queue = append(queue, []byte(kv.val))
kv, err = b.tx.stm.Next(bucketPrefix, kv.key)
if err != nil {
return err
}
}
// Finally delete the sequence key for the bucket.
b.tx.stm.Del(string(makeSequenceKey(id)))
}
// Delete the top level bucket.
b.tx.del(bucketKey)
// Delete the top level bucket and sequence key.
b.tx.stm.Del(bucketKey)
b.tx.stm.Del(string(makeSequenceKey(bucketVal)))
return nil
}
@@ -292,8 +272,17 @@ func (b *readWriteBucket) Put(key, value []byte) error {
return walletdb.ErrKeyRequired
}
val, err := b.tx.stm.Get(string(makeBucketKey(b.id, key)))
if err != nil {
return err
}
if val != nil {
return walletdb.ErrIncompatibleValue
}
// Update the transaction with the new value.
b.tx.put(string(makeValueKey(b.id, key)), string(value))
b.tx.stm.Put(string(makeValueKey(b.id, key)), string(value))
return nil
}
@@ -306,7 +295,7 @@ func (b *readWriteBucket) Delete(key []byte) error {
}
// Update the transaction to delete the key/value.
b.tx.del(string(makeValueKey(b.id, key)))
b.tx.stm.Del(string(makeValueKey(b.id, key)))
return nil
}
@@ -336,7 +325,7 @@ func (b *readWriteBucket) SetSequence(v uint64) error {
val := strconv.FormatUint(v, 10)
// Update the transaction with the new value for the sequence key.
b.tx.put(string(makeSequenceKey(b.id)), val)
b.tx.stm.Put(string(makeSequenceKey(b.id)), val)
return nil
}
+222 -103
View File
@@ -8,7 +8,7 @@ import (
"testing"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBucketCreation(t *testing.T) {
@@ -18,70 +18,70 @@ func TestBucketCreation(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
// empty bucket name
b, err := tx.CreateTopLevelBucket(nil)
assert.Error(t, walletdb.ErrBucketNameRequired, err)
assert.Nil(t, b)
require.Error(t, walletdb.ErrBucketNameRequired, err)
require.Nil(t, b)
// empty bucket name
b, err = tx.CreateTopLevelBucket([]byte(""))
assert.Error(t, walletdb.ErrBucketNameRequired, err)
assert.Nil(t, b)
require.Error(t, walletdb.ErrBucketNameRequired, err)
require.Nil(t, b)
// "apple"
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.NoError(t, err)
assert.NotNil(t, apple)
require.NoError(t, err)
require.NotNil(t, apple)
// Check bucket tx.
assert.Equal(t, tx, apple.Tx())
require.Equal(t, tx, apple.Tx())
// "apple" already created
b, err = tx.CreateTopLevelBucket([]byte("apple"))
assert.NoError(t, err)
assert.NotNil(t, b)
require.NoError(t, err)
require.NotNil(t, b)
// "apple/banana"
banana, err := apple.CreateBucket([]byte("banana"))
assert.NoError(t, err)
assert.NotNil(t, banana)
require.NoError(t, err)
require.NotNil(t, banana)
banana, err = apple.CreateBucketIfNotExists([]byte("banana"))
assert.NoError(t, err)
assert.NotNil(t, banana)
require.NoError(t, err)
require.NotNil(t, banana)
// Try creating "apple/banana" again
b, err = apple.CreateBucket([]byte("banana"))
assert.Error(t, walletdb.ErrBucketExists, err)
assert.Nil(t, b)
require.Error(t, walletdb.ErrBucketExists, err)
require.Nil(t, b)
// "apple/mango"
mango, err := apple.CreateBucket([]byte("mango"))
assert.Nil(t, err)
assert.NotNil(t, mango)
require.Nil(t, err)
require.NotNil(t, mango)
// "apple/banana/pear"
pear, err := banana.CreateBucket([]byte("pear"))
assert.Nil(t, err)
assert.NotNil(t, pear)
require.Nil(t, err)
require.NotNil(t, pear)
// empty bucket
assert.Nil(t, apple.NestedReadWriteBucket(nil))
assert.Nil(t, apple.NestedReadWriteBucket([]byte("")))
require.Nil(t, apple.NestedReadWriteBucket(nil))
require.Nil(t, apple.NestedReadWriteBucket([]byte("")))
// "apple/pear" doesn't exist
assert.Nil(t, apple.NestedReadWriteBucket([]byte("pear")))
require.Nil(t, apple.NestedReadWriteBucket([]byte("pear")))
// "apple/banana" exits
assert.NotNil(t, apple.NestedReadWriteBucket([]byte("banana")))
assert.NotNil(t, apple.NestedReadBucket([]byte("banana")))
require.NotNil(t, apple.NestedReadWriteBucket([]byte("banana")))
require.NotNil(t, apple.NestedReadBucket([]byte("banana")))
return nil
})
assert.Nil(t, err)
require.Nil(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
@@ -89,7 +89,7 @@ func TestBucketCreation(t *testing.T) {
bkey("apple", "mango"): bval("apple", "mango"),
bkey("apple", "banana", "pear"): bval("apple", "banana", "pear"),
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
func TestBucketDeletion(t *testing.T) {
@@ -99,99 +99,99 @@ func TestBucketDeletion(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
// "apple"
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.Nil(t, err)
assert.NotNil(t, apple)
require.Nil(t, err)
require.NotNil(t, apple)
// "apple/banana"
banana, err := apple.CreateBucket([]byte("banana"))
assert.Nil(t, err)
assert.NotNil(t, banana)
require.Nil(t, err)
require.NotNil(t, banana)
kvs := []KV{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}}
for _, kv := range kvs {
assert.NoError(t, banana.Put([]byte(kv.key), []byte(kv.val)))
assert.Equal(t, []byte(kv.val), banana.Get([]byte(kv.key)))
require.NoError(t, banana.Put([]byte(kv.key), []byte(kv.val)))
require.Equal(t, []byte(kv.val), banana.Get([]byte(kv.key)))
}
// Delete a k/v from "apple/banana"
assert.NoError(t, banana.Delete([]byte("key2")))
require.NoError(t, banana.Delete([]byte("key2")))
// Try getting/putting/deleting invalid k/v's.
assert.Nil(t, banana.Get(nil))
assert.Error(t, walletdb.ErrKeyRequired, banana.Put(nil, []byte("val")))
assert.Error(t, walletdb.ErrKeyRequired, banana.Delete(nil))
require.Nil(t, banana.Get(nil))
require.Error(t, walletdb.ErrKeyRequired, banana.Put(nil, []byte("val")))
require.Error(t, walletdb.ErrKeyRequired, banana.Delete(nil))
// Try deleting a k/v that doesn't exist.
assert.NoError(t, banana.Delete([]byte("nokey")))
require.NoError(t, banana.Delete([]byte("nokey")))
// "apple/pear"
pear, err := apple.CreateBucket([]byte("pear"))
assert.Nil(t, err)
assert.NotNil(t, pear)
require.Nil(t, err)
require.NotNil(t, pear)
// Put some values into "apple/pear"
for _, kv := range kvs {
assert.Nil(t, pear.Put([]byte(kv.key), []byte(kv.val)))
assert.Equal(t, []byte(kv.val), pear.Get([]byte(kv.key)))
require.Nil(t, pear.Put([]byte(kv.key), []byte(kv.val)))
require.Equal(t, []byte(kv.val), pear.Get([]byte(kv.key)))
}
// Create nested bucket "apple/pear/cherry"
cherry, err := pear.CreateBucket([]byte("cherry"))
assert.Nil(t, err)
assert.NotNil(t, cherry)
require.Nil(t, err)
require.NotNil(t, cherry)
// Put some values into "apple/pear/cherry"
for _, kv := range kvs {
assert.NoError(t, cherry.Put([]byte(kv.key), []byte(kv.val)))
require.NoError(t, cherry.Put([]byte(kv.key), []byte(kv.val)))
}
// Read back values in "apple/pear/cherry" trough a read bucket.
cherryReadBucket := pear.NestedReadBucket([]byte("cherry"))
for _, kv := range kvs {
assert.Equal(
require.Equal(
t, []byte(kv.val),
cherryReadBucket.Get([]byte(kv.key)),
)
}
// Try deleting some invalid buckets.
assert.Error(t,
require.Error(t,
walletdb.ErrBucketNameRequired, apple.DeleteNestedBucket(nil),
)
// Try deleting a non existing bucket.
assert.Error(
require.Error(
t,
walletdb.ErrBucketNotFound,
apple.DeleteNestedBucket([]byte("missing")),
)
// Delete "apple/pear"
assert.Nil(t, apple.DeleteNestedBucket([]byte("pear")))
require.Nil(t, apple.DeleteNestedBucket([]byte("pear")))
// "apple/pear" deleted
assert.Nil(t, apple.NestedReadWriteBucket([]byte("pear")))
require.Nil(t, apple.NestedReadWriteBucket([]byte("pear")))
// "apple/pear/cherry" deleted
assert.Nil(t, pear.NestedReadWriteBucket([]byte("cherry")))
require.Nil(t, pear.NestedReadWriteBucket([]byte("cherry")))
// Values deleted too.
for _, kv := range kvs {
assert.Nil(t, pear.Get([]byte(kv.key)))
assert.Nil(t, cherry.Get([]byte(kv.key)))
require.Nil(t, pear.Get([]byte(kv.key)))
require.Nil(t, cherry.Get([]byte(kv.key)))
}
// "aple/banana" exists
assert.NotNil(t, apple.NestedReadWriteBucket([]byte("banana")))
require.NotNil(t, apple.NestedReadWriteBucket([]byte("banana")))
return nil
})
assert.Nil(t, err)
require.Nil(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
@@ -199,7 +199,7 @@ func TestBucketDeletion(t *testing.T) {
vkey("key1", "apple", "banana"): "val1",
vkey("key3", "apple", "banana"): "val3",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
func TestBucketForEach(t *testing.T) {
@@ -209,28 +209,28 @@ func TestBucketForEach(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
// "apple"
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.Nil(t, err)
assert.NotNil(t, apple)
require.Nil(t, err)
require.NotNil(t, apple)
// "apple/banana"
banana, err := apple.CreateBucket([]byte("banana"))
assert.Nil(t, err)
assert.NotNil(t, banana)
require.Nil(t, err)
require.NotNil(t, banana)
kvs := []KV{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}}
// put some values into "apple" and "apple/banana" too
for _, kv := range kvs {
assert.Nil(t, apple.Put([]byte(kv.key), []byte(kv.val)))
assert.Equal(t, []byte(kv.val), apple.Get([]byte(kv.key)))
require.Nil(t, apple.Put([]byte(kv.key), []byte(kv.val)))
require.Equal(t, []byte(kv.val), apple.Get([]byte(kv.key)))
assert.Nil(t, banana.Put([]byte(kv.key), []byte(kv.val)))
assert.Equal(t, []byte(kv.val), banana.Get([]byte(kv.key)))
require.Nil(t, banana.Put([]byte(kv.key), []byte(kv.val)))
require.Equal(t, []byte(kv.val), banana.Get([]byte(kv.key)))
}
got := make(map[string]string)
@@ -246,8 +246,8 @@ func TestBucketForEach(t *testing.T) {
"banana": "",
}
assert.NoError(t, err)
assert.Equal(t, expected, got)
require.NoError(t, err)
require.Equal(t, expected, got)
got = make(map[string]string)
err = banana.ForEach(func(key, val []byte) error {
@@ -255,15 +255,15 @@ func TestBucketForEach(t *testing.T) {
return nil
})
assert.NoError(t, err)
require.NoError(t, err)
// remove the sub-bucket key
delete(expected, "banana")
assert.Equal(t, expected, got)
require.Equal(t, expected, got)
return nil
})
assert.Nil(t, err)
require.Nil(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
@@ -275,7 +275,7 @@ func TestBucketForEach(t *testing.T) {
vkey("key2", "apple", "banana"): "val2",
vkey("key3", "apple", "banana"): "val3",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
func TestBucketForEachWithError(t *testing.T) {
@@ -285,37 +285,37 @@ func TestBucketForEachWithError(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
// "apple"
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.Nil(t, err)
assert.NotNil(t, apple)
require.Nil(t, err)
require.NotNil(t, apple)
// "apple/banana"
banana, err := apple.CreateBucket([]byte("banana"))
assert.Nil(t, err)
assert.NotNil(t, banana)
require.Nil(t, err)
require.NotNil(t, banana)
// "apple/pear"
pear, err := apple.CreateBucket([]byte("pear"))
assert.Nil(t, err)
assert.NotNil(t, pear)
require.Nil(t, err)
require.NotNil(t, pear)
kvs := []KV{{"key1", "val1"}, {"key2", "val2"}}
// Put some values into "apple" and "apple/banana" too.
for _, kv := range kvs {
assert.Nil(t, apple.Put([]byte(kv.key), []byte(kv.val)))
assert.Equal(t, []byte(kv.val), apple.Get([]byte(kv.key)))
require.Nil(t, apple.Put([]byte(kv.key), []byte(kv.val)))
require.Equal(t, []byte(kv.val), apple.Get([]byte(kv.key)))
}
got := make(map[string]string)
i := 0
// Error while iterating value keys.
err = apple.ForEach(func(key, val []byte) error {
if i == 1 {
if i == 2 {
return fmt.Errorf("error")
}
@@ -325,11 +325,12 @@ func TestBucketForEachWithError(t *testing.T) {
})
expected := map[string]string{
"key1": "val1",
"banana": "",
"key1": "val1",
}
assert.Equal(t, expected, got)
assert.Error(t, err)
require.Equal(t, expected, got)
require.Error(t, err)
got = make(map[string]string)
i = 0
@@ -345,17 +346,17 @@ func TestBucketForEachWithError(t *testing.T) {
})
expected = map[string]string{
"banana": "",
"key1": "val1",
"key2": "val2",
"banana": "",
}
assert.Equal(t, expected, got)
assert.Error(t, err)
require.Equal(t, expected, got)
require.Error(t, err)
return nil
})
assert.Nil(t, err)
require.Nil(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
@@ -364,7 +365,7 @@ func TestBucketForEachWithError(t *testing.T) {
vkey("key1", "apple"): "val1",
vkey("key2", "apple"): "val2",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
func TestBucketSequence(t *testing.T) {
@@ -374,31 +375,149 @@ func TestBucketSequence(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.Nil(t, err)
assert.NotNil(t, apple)
require.Nil(t, err)
require.NotNil(t, apple)
banana, err := apple.CreateBucket([]byte("banana"))
assert.Nil(t, err)
assert.NotNil(t, banana)
require.Nil(t, err)
require.NotNil(t, banana)
assert.Equal(t, uint64(0), apple.Sequence())
assert.Equal(t, uint64(0), banana.Sequence())
require.Equal(t, uint64(0), apple.Sequence())
require.Equal(t, uint64(0), banana.Sequence())
assert.Nil(t, apple.SetSequence(math.MaxUint64))
assert.Equal(t, uint64(math.MaxUint64), apple.Sequence())
require.Nil(t, apple.SetSequence(math.MaxUint64))
require.Equal(t, uint64(math.MaxUint64), apple.Sequence())
for i := uint64(0); i < uint64(5); i++ {
s, err := apple.NextSequence()
assert.Nil(t, err)
assert.Equal(t, i, s)
require.Nil(t, err)
require.Equal(t, i, s)
}
return nil
})
assert.Nil(t, err)
require.Nil(t, err)
}
// TestKeyClash tests that one cannot create a bucket if a value with the same
// key exists and the same is true in reverse: that a value cannot be put if
// a bucket with the same key exists.
func TestKeyClash(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
require.NoError(t, err)
// First:
// put: /apple/key -> val
// create bucket: /apple/banana
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
require.Nil(t, err)
require.NotNil(t, apple)
require.NoError(t, apple.Put([]byte("key"), []byte("val")))
banana, err := apple.CreateBucket([]byte("banana"))
require.Nil(t, err)
require.NotNil(t, banana)
return nil
})
require.Nil(t, err)
// Next try to:
// put: /apple/banana -> val => will fail (as /apple/banana is a bucket)
// create bucket: /apple/key => will fail (as /apple/key is a value)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
require.Nil(t, err)
require.NotNil(t, apple)
require.Error(t,
walletdb.ErrIncompatibleValue,
apple.Put([]byte("banana"), []byte("val")),
)
b, err := apple.CreateBucket([]byte("key"))
require.Nil(t, b)
require.Error(t, walletdb.ErrIncompatibleValue, b)
b, err = apple.CreateBucketIfNotExists([]byte("key"))
require.Nil(t, b)
require.Error(t, walletdb.ErrIncompatibleValue, b)
return nil
})
require.Nil(t, err)
// Except that the only existing items in the db are:
// bucket: /apple
// bucket: /apple/banana
// value: /apple/key -> val
expected := map[string]string{
bkey("apple"): bval("apple"),
bkey("apple", "banana"): bval("apple", "banana"),
vkey("key", "apple"): "val",
}
require.Equal(t, expected, f.Dump())
}
// TestBucketCreateDelete tests that creating then deleting then creating a
// bucket suceeds.
func TestBucketCreateDelete(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
require.NoError(t, err)
require.NotNil(t, apple)
banana, err := apple.CreateBucket([]byte("banana"))
require.NoError(t, err)
require.NotNil(t, banana)
return nil
})
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple := tx.ReadWriteBucket([]byte("apple"))
require.NotNil(t, apple)
require.NoError(t, apple.DeleteNestedBucket([]byte("banana")))
return nil
})
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple := tx.ReadWriteBucket([]byte("apple"))
require.NotNil(t, apple)
require.NoError(t, apple.Put([]byte("banana"), []byte("value")))
return nil
})
require.NoError(t, err)
expected := map[string]string{
vkey("banana", "apple"): "value",
bkey("apple"): bval("apple"),
}
require.Equal(t, expected, f.Dump())
}
+12 -14
View File
@@ -19,7 +19,7 @@ type readWriteCursor struct {
func newReadWriteCursor(bucket *readWriteBucket) *readWriteCursor {
return &readWriteCursor{
bucket: bucket,
prefix: string(makeValuePrefix(bucket.id)),
prefix: string(bucket.id),
}
}
@@ -35,8 +35,7 @@ func (c *readWriteCursor) First() (key, value []byte) {
if kv != nil {
c.currKey = kv.key
// Chop the prefix and return the key/value.
return []byte(kv.key[len(c.prefix):]), []byte(kv.val)
return getKeyVal(kv)
}
return nil, nil
@@ -53,8 +52,7 @@ func (c *readWriteCursor) Last() (key, value []byte) {
if kv != nil {
c.currKey = kv.key
// Chop the prefix and return the key/value.
return []byte(kv.key[len(c.prefix):]), []byte(kv.val)
return getKeyVal(kv)
}
return nil, nil
@@ -71,8 +69,7 @@ func (c *readWriteCursor) Next() (key, value []byte) {
if kv != nil {
c.currKey = kv.key
// Chop the prefix and return the key/value.
return []byte(kv.key[len(c.prefix):]), []byte(kv.val)
return getKeyVal(kv)
}
return nil, nil
@@ -89,8 +86,7 @@ func (c *readWriteCursor) Prev() (key, value []byte) {
if kv != nil {
c.currKey = kv.key
// Chop the prefix and return the key/value.
return []byte(kv.key[len(c.prefix):]), []byte(kv.val)
return getKeyVal(kv)
}
return nil, nil
@@ -115,8 +111,7 @@ func (c *readWriteCursor) Seek(seek []byte) (key, value []byte) {
if kv != nil {
c.currKey = kv.key
// Chop the prefix and return the key/value.
return []byte(kv.key[len(c.prefix):]), []byte(kv.val)
return getKeyVal(kv)
}
return nil, nil
@@ -133,11 +128,14 @@ func (c *readWriteCursor) Delete() error {
return err
}
// Delete the current key.
c.bucket.tx.stm.Del(c.currKey)
if isBucketKey(c.currKey) {
c.bucket.DeleteNestedBucket(getKey(c.currKey))
} else {
c.bucket.Delete(getKey(c.currKey))
}
// Set current key to the next one if possible.
if nextKey != nil {
// Set current key to the next one.
c.currKey = nextKey.key
}
+149 -74
View File
@@ -6,7 +6,7 @@ import (
"testing"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestReadCursorEmptyInterval(t *testing.T) {
@@ -16,41 +16,41 @@ func TestReadCursorEmptyInterval(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
err = db.Update(func(tx walletdb.ReadWriteTx) error {
b, err := tx.CreateTopLevelBucket([]byte("alma"))
assert.NoError(t, err)
assert.NotNil(t, b)
b, err := tx.CreateTopLevelBucket([]byte("apple"))
require.NoError(t, err)
require.NotNil(t, b)
return nil
})
assert.NoError(t, err)
require.NoError(t, err)
err = db.View(func(tx walletdb.ReadTx) error {
b := tx.ReadBucket([]byte("alma"))
assert.NotNil(t, b)
b := tx.ReadBucket([]byte("apple"))
require.NotNil(t, b)
cursor := b.ReadCursor()
k, v := cursor.First()
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
k, v = cursor.Next()
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
k, v = cursor.Last()
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
k, v = cursor.Prev()
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
return nil
})
assert.NoError(t, err)
require.NoError(t, err)
}
func TestReadCursorNonEmptyInterval(t *testing.T) {
@@ -60,7 +60,7 @@ func TestReadCursorNonEmptyInterval(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
testKeyValues := []KV{
{"b", "1"},
@@ -70,21 +70,21 @@ func TestReadCursorNonEmptyInterval(t *testing.T) {
}
err = db.Update(func(tx walletdb.ReadWriteTx) error {
b, err := tx.CreateTopLevelBucket([]byte("alma"))
assert.NoError(t, err)
assert.NotNil(t, b)
b, err := tx.CreateTopLevelBucket([]byte("apple"))
require.NoError(t, err)
require.NotNil(t, b)
for _, kv := range testKeyValues {
assert.NoError(t, b.Put([]byte(kv.key), []byte(kv.val)))
require.NoError(t, b.Put([]byte(kv.key), []byte(kv.val)))
}
return nil
})
assert.NoError(t, err)
require.NoError(t, err)
err = db.View(func(tx walletdb.ReadTx) error {
b := tx.ReadBucket([]byte("alma"))
assert.NotNil(t, b)
b := tx.ReadBucket([]byte("apple"))
require.NotNil(t, b)
// Iterate from the front.
var kvs []KV
@@ -95,7 +95,7 @@ func TestReadCursorNonEmptyInterval(t *testing.T) {
kvs = append(kvs, KV{string(k), string(v)})
k, v = cursor.Next()
}
assert.Equal(t, testKeyValues, kvs)
require.Equal(t, testKeyValues, kvs)
// Iterate from the back.
kvs = []KV{}
@@ -105,29 +105,29 @@ func TestReadCursorNonEmptyInterval(t *testing.T) {
kvs = append(kvs, KV{string(k), string(v)})
k, v = cursor.Prev()
}
assert.Equal(t, reverseKVs(testKeyValues), kvs)
require.Equal(t, reverseKVs(testKeyValues), kvs)
// Random access
perm := []int{3, 0, 2, 1}
for _, i := range perm {
k, v := cursor.Seek([]byte(testKeyValues[i].key))
assert.Equal(t, []byte(testKeyValues[i].key), k)
assert.Equal(t, []byte(testKeyValues[i].val), v)
require.Equal(t, []byte(testKeyValues[i].key), k)
require.Equal(t, []byte(testKeyValues[i].val), v)
}
// Seek to nonexisting key.
k, v = cursor.Seek(nil)
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
k, v = cursor.Seek([]byte("x"))
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
return nil
})
assert.NoError(t, err)
require.NoError(t, err)
}
func TestReadWriteCursor(t *testing.T) {
@@ -137,7 +137,7 @@ func TestReadWriteCursor(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
testKeyValues := []KV{
{"b", "1"},
@@ -149,24 +149,24 @@ func TestReadWriteCursor(t *testing.T) {
count := len(testKeyValues)
// Pre-store the first half of the interval.
assert.NoError(t, db.Update(func(tx walletdb.ReadWriteTx) error {
require.NoError(t, db.Update(func(tx walletdb.ReadWriteTx) error {
b, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.NoError(t, err)
assert.NotNil(t, b)
require.NoError(t, err)
require.NotNil(t, b)
for i := 0; i < count/2; i++ {
err = b.Put(
[]byte(testKeyValues[i].key),
[]byte(testKeyValues[i].val),
)
assert.NoError(t, err)
require.NoError(t, err)
}
return nil
}))
err = db.Update(func(tx walletdb.ReadWriteTx) error {
b := tx.ReadWriteBucket([]byte("apple"))
assert.NotNil(t, b)
require.NotNil(t, b)
// Store the second half of the interval.
for i := count / 2; i < count; i++ {
@@ -174,77 +174,77 @@ func TestReadWriteCursor(t *testing.T) {
[]byte(testKeyValues[i].key),
[]byte(testKeyValues[i].val),
)
assert.NoError(t, err)
require.NoError(t, err)
}
cursor := b.ReadWriteCursor()
// First on valid interval.
fk, fv := cursor.First()
assert.Equal(t, []byte("b"), fk)
assert.Equal(t, []byte("1"), fv)
require.Equal(t, []byte("b"), fk)
require.Equal(t, []byte("1"), fv)
// Prev(First()) = nil
k, v := cursor.Prev()
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
// Last on valid interval.
lk, lv := cursor.Last()
assert.Equal(t, []byte("e"), lk)
assert.Equal(t, []byte("4"), lv)
require.Equal(t, []byte("e"), lk)
require.Equal(t, []byte("4"), lv)
// Next(Last()) = nil
k, v = cursor.Next()
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
// Delete first item, then add an item before the
// deleted one. Check that First/Next will "jump"
// over the deleted item and return the new first.
_, _ = cursor.First()
assert.NoError(t, cursor.Delete())
assert.NoError(t, b.Put([]byte("a"), []byte("0")))
require.NoError(t, cursor.Delete())
require.NoError(t, b.Put([]byte("a"), []byte("0")))
fk, fv = cursor.First()
assert.Equal(t, []byte("a"), fk)
assert.Equal(t, []byte("0"), fv)
require.Equal(t, []byte("a"), fk)
require.Equal(t, []byte("0"), fv)
k, v = cursor.Next()
assert.Equal(t, []byte("c"), k)
assert.Equal(t, []byte("2"), v)
require.Equal(t, []byte("c"), k)
require.Equal(t, []byte("2"), v)
// Similarly test that a new end is returned if
// the old end is deleted first.
_, _ = cursor.Last()
assert.NoError(t, cursor.Delete())
assert.NoError(t, b.Put([]byte("f"), []byte("5")))
require.NoError(t, cursor.Delete())
require.NoError(t, b.Put([]byte("f"), []byte("5")))
lk, lv = cursor.Last()
assert.Equal(t, []byte("f"), lk)
assert.Equal(t, []byte("5"), lv)
require.Equal(t, []byte("f"), lk)
require.Equal(t, []byte("5"), lv)
k, v = cursor.Prev()
assert.Equal(t, []byte("da"), k)
assert.Equal(t, []byte("3"), v)
require.Equal(t, []byte("da"), k)
require.Equal(t, []byte("3"), v)
// Overwrite k/v in the middle of the interval.
assert.NoError(t, b.Put([]byte("c"), []byte("3")))
require.NoError(t, b.Put([]byte("c"), []byte("3")))
k, v = cursor.Prev()
assert.Equal(t, []byte("c"), k)
assert.Equal(t, []byte("3"), v)
require.Equal(t, []byte("c"), k)
require.Equal(t, []byte("3"), v)
// Insert new key/values.
assert.NoError(t, b.Put([]byte("cx"), []byte("x")))
assert.NoError(t, b.Put([]byte("cy"), []byte("y")))
require.NoError(t, b.Put([]byte("cx"), []byte("x")))
require.NoError(t, b.Put([]byte("cy"), []byte("y")))
k, v = cursor.Next()
assert.Equal(t, []byte("cx"), k)
assert.Equal(t, []byte("x"), v)
require.Equal(t, []byte("cx"), k)
require.Equal(t, []byte("x"), v)
k, v = cursor.Next()
assert.Equal(t, []byte("cy"), k)
assert.Equal(t, []byte("y"), v)
require.Equal(t, []byte("cy"), k)
require.Equal(t, []byte("y"), v)
expected := []KV{
{"a", "0"},
@@ -263,7 +263,7 @@ func TestReadWriteCursor(t *testing.T) {
kvs = append(kvs, KV{string(k), string(v)})
k, v = cursor.Next()
}
assert.Equal(t, expected, kvs)
require.Equal(t, expected, kvs)
// Iterate from the back.
kvs = []KV{}
@@ -273,12 +273,12 @@ func TestReadWriteCursor(t *testing.T) {
kvs = append(kvs, KV{string(k), string(v)})
k, v = cursor.Prev()
}
assert.Equal(t, reverseKVs(expected), kvs)
require.Equal(t, reverseKVs(expected), kvs)
return nil
})
assert.NoError(t, err)
require.NoError(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
@@ -289,5 +289,80 @@ func TestReadWriteCursor(t *testing.T) {
vkey("da", "apple"): "3",
vkey("f", "apple"): "5",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
// TestReadWriteCursorWithBucketAndValue tests that cursors are able to iterate
// over both bucket and value keys if both are present in the iterated bucket.
func TestReadWriteCursorWithBucketAndValue(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
require.NoError(t, err)
// Pre-store the first half of the interval.
require.NoError(t, db.Update(func(tx walletdb.ReadWriteTx) error {
b, err := tx.CreateTopLevelBucket([]byte("apple"))
require.NoError(t, err)
require.NotNil(t, b)
require.NoError(t, b.Put([]byte("key"), []byte("val")))
b1, err := b.CreateBucket([]byte("banana"))
require.NoError(t, err)
require.NotNil(t, b1)
b2, err := b.CreateBucket([]byte("pear"))
require.NoError(t, err)
require.NotNil(t, b2)
return nil
}))
err = db.View(func(tx walletdb.ReadTx) error {
b := tx.ReadBucket([]byte("apple"))
require.NotNil(t, b)
cursor := b.ReadCursor()
// First on valid interval.
k, v := cursor.First()
require.Equal(t, []byte("banana"), k)
require.Nil(t, v)
k, v = cursor.Next()
require.Equal(t, []byte("key"), k)
require.Equal(t, []byte("val"), v)
k, v = cursor.Last()
require.Equal(t, []byte("pear"), k)
require.Nil(t, v)
k, v = cursor.Seek([]byte("k"))
require.Equal(t, []byte("key"), k)
require.Equal(t, []byte("val"), v)
k, v = cursor.Seek([]byte("banana"))
require.Equal(t, []byte("banana"), k)
require.Nil(t, v)
k, v = cursor.Next()
require.Equal(t, []byte("key"), k)
require.Equal(t, []byte("val"), v)
return nil
})
require.NoError(t, err)
expected := map[string]string{
bkey("apple"): bval("apple"),
bkey("apple", "banana"): bval("apple", "banana"),
bkey("apple", "pear"): bval("apple", "pear"),
vkey("key", "apple"): "val",
}
require.Equal(t, expected, f.Dump())
}
-53
View File
@@ -17,14 +17,6 @@ type readWriteTx struct {
// active is true if the transaction hasn't been committed yet.
active bool
// dirty is true if we intent to update a value in this transaction.
dirty bool
// lset holds key/value set that we want to lock on. If upon commit the
// transaction is dirty and the lset is not empty, we'll bump the mod
// version of these key/values.
lset map[string]string
}
// newReadWriteTx creates an rw transaction with the passed STM.
@@ -33,7 +25,6 @@ func newReadWriteTx(stm STM, prefix string) *readWriteTx {
stm: stm,
active: true,
rootBucketID: makeBucketID([]byte(prefix)),
lset: make(map[string]string),
}
}
@@ -43,50 +34,6 @@ func rootBucket(tx *readWriteTx) *readWriteBucket {
return newReadWriteBucket(tx, tx.rootBucketID[:], tx.rootBucketID[:])
}
// lock adds a key value to the lock set.
func (tx *readWriteTx) lock(key, val string) {
tx.stm.Lock(key)
if !tx.dirty {
tx.lset[key] = val
} else {
// Bump the mod version of the key,
// leaving the value intact.
tx.stm.Put(key, val)
}
}
// put updates the passed key/value.
func (tx *readWriteTx) put(key, val string) {
tx.stm.Put(key, val)
tx.setDirty()
}
// del marks the passed key deleted.
func (tx *readWriteTx) del(key string) {
tx.stm.Del(key)
tx.setDirty()
}
// setDirty marks the transaction dirty and bumps
// mod version for the existing lock set if it is
// not empty.
func (tx *readWriteTx) setDirty() {
// Bump the lock set.
if !tx.dirty && len(tx.lset) > 0 {
for key, val := range tx.lset {
// Bump the mod version of the key,
// leaving the value intact.
tx.stm.Put(key, val)
}
// Clear the lock set.
tx.lset = make(map[string]string)
}
// Set dirty.
tx.dirty = true
}
// ReadBucket opens the root bucket for read only access. If the bucket
// described by the key does not exist, nil is returned.
func (tx *readWriteTx) ReadBucket(key []byte) walletdb.ReadBucket {
+45 -45
View File
@@ -6,7 +6,7 @@ import (
"testing"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestTxManualCommit(t *testing.T) {
@@ -16,11 +16,11 @@ func TestTxManualCommit(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
tx, err := db.BeginReadWriteTx()
assert.NoError(t, err)
assert.NotNil(t, tx)
require.NoError(t, err)
require.NotNil(t, tx)
committed := false
@@ -29,24 +29,24 @@ func TestTxManualCommit(t *testing.T) {
})
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.NoError(t, err)
assert.NotNil(t, apple)
assert.NoError(t, apple.Put([]byte("testKey"), []byte("testVal")))
require.NoError(t, err)
require.NotNil(t, apple)
require.NoError(t, apple.Put([]byte("testKey"), []byte("testVal")))
banana, err := tx.CreateTopLevelBucket([]byte("banana"))
assert.NoError(t, err)
assert.NotNil(t, banana)
assert.NoError(t, banana.Put([]byte("testKey"), []byte("testVal")))
assert.NoError(t, tx.DeleteTopLevelBucket([]byte("banana")))
require.NoError(t, err)
require.NotNil(t, banana)
require.NoError(t, banana.Put([]byte("testKey"), []byte("testVal")))
require.NoError(t, tx.DeleteTopLevelBucket([]byte("banana")))
assert.NoError(t, tx.Commit())
assert.True(t, committed)
require.NoError(t, tx.Commit())
require.True(t, committed)
expected := map[string]string{
bkey("apple"): bval("apple"),
vkey("testKey", "apple"): "testVal",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
func TestTxRollback(t *testing.T) {
@@ -56,21 +56,21 @@ func TestTxRollback(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
tx, err := db.BeginReadWriteTx()
assert.Nil(t, err)
assert.NotNil(t, tx)
require.Nil(t, err)
require.NotNil(t, tx)
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.Nil(t, err)
assert.NotNil(t, apple)
require.Nil(t, err)
require.NotNil(t, apple)
assert.NoError(t, apple.Put([]byte("testKey"), []byte("testVal")))
require.NoError(t, apple.Put([]byte("testKey"), []byte("testVal")))
assert.NoError(t, tx.Rollback())
assert.Error(t, walletdb.ErrTxClosed, tx.Commit())
assert.Equal(t, map[string]string{}, f.Dump())
require.NoError(t, tx.Rollback())
require.Error(t, walletdb.ErrTxClosed, tx.Commit())
require.Equal(t, map[string]string{}, f.Dump())
}
func TestChangeDuringManualTx(t *testing.T) {
@@ -80,24 +80,24 @@ func TestChangeDuringManualTx(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
tx, err := db.BeginReadWriteTx()
assert.Nil(t, err)
assert.NotNil(t, tx)
require.Nil(t, err)
require.NotNil(t, tx)
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.Nil(t, err)
assert.NotNil(t, apple)
require.Nil(t, err)
require.NotNil(t, apple)
assert.NoError(t, apple.Put([]byte("testKey"), []byte("testVal")))
require.NoError(t, apple.Put([]byte("testKey"), []byte("testVal")))
// Try overwriting the bucket key.
f.Put(bkey("apple"), "banana")
// TODO: translate error
assert.NotNil(t, tx.Commit())
assert.Equal(t, map[string]string{
require.NotNil(t, tx.Commit())
require.Equal(t, map[string]string{
bkey("apple"): "banana",
}, f.Dump())
}
@@ -109,16 +109,16 @@ func TestChangeDuringUpdate(t *testing.T) {
defer f.Cleanup()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
count := 0
err = db.Update(func(tx walletdb.ReadWriteTx) error {
apple, err := tx.CreateTopLevelBucket([]byte("apple"))
assert.NoError(t, err)
assert.NotNil(t, apple)
require.NoError(t, err)
require.NotNil(t, apple)
assert.NoError(t, apple.Put([]byte("key"), []byte("value")))
require.NoError(t, apple.Put([]byte("key"), []byte("value")))
if count == 0 {
f.Put(vkey("key", "apple"), "new_value")
@@ -127,30 +127,30 @@ func TestChangeDuringUpdate(t *testing.T) {
cursor := apple.ReadCursor()
k, v := cursor.First()
assert.Equal(t, []byte("key"), k)
assert.Equal(t, []byte("value"), v)
assert.Equal(t, v, apple.Get([]byte("key")))
require.Equal(t, []byte("key"), k)
require.Equal(t, []byte("value"), v)
require.Equal(t, v, apple.Get([]byte("key")))
k, v = cursor.Next()
if count == 0 {
assert.Nil(t, k)
assert.Nil(t, v)
require.Nil(t, k)
require.Nil(t, v)
} else {
assert.Equal(t, []byte("key2"), k)
assert.Equal(t, []byte("value2"), v)
require.Equal(t, []byte("key2"), k)
require.Equal(t, []byte("value2"), v)
}
count++
return nil
})
assert.Nil(t, err)
assert.Equal(t, count, 2)
require.Nil(t, err)
require.Equal(t, count, 2)
expected := map[string]string{
bkey("apple"): bval("apple"),
vkey("key", "apple"): "value",
vkey("key2", "apple"): "value2",
}
assert.Equal(t, expected, f.Dump())
require.Equal(t, expected, f.Dump())
}
+99 -75
View File
@@ -32,11 +32,6 @@ type STM interface {
// set. Returns nil if there's no matching key, or the key is empty.
Get(key string) ([]byte, error)
// Lock adds a key to the lock set. If the lock set is not empty, we'll
// only check for conflicts in the lock set and the write set, instead
// of all read keys plus the write set.
Lock(key string)
// Put adds a value for a key to the txn's write set.
Put(key, val string)
@@ -139,6 +134,10 @@ type stm struct {
// execute in the STM run loop.
manual bool
// txQueue is lightweight contention manager, which is used to detect
// transaction conflicts and reduce retries.
txQueue *commitQueue
// options stores optional settings passed by the user.
options *STMOptions
@@ -151,9 +150,6 @@ type stm struct {
// wset holds overwritten keys and their values.
wset writeSet
// lset holds keys we intent to lock on.
lset map[string]interface{}
// getOpts are the opts used for gets.
getOpts []v3.OpOption
@@ -191,18 +187,22 @@ func WithCommitStatsCallback(cb func(bool, CommitStats)) STMOptionFunc {
// RunSTM runs the apply function by creating an STM using serializable snapshot
// isolation, passing it to the apply and handling commit errors and retries.
func RunSTM(cli *v3.Client, apply func(STM) error, so ...STMOptionFunc) error {
return runSTM(makeSTM(cli, false, so...), apply)
func RunSTM(cli *v3.Client, apply func(STM) error, txQueue *commitQueue,
so ...STMOptionFunc) error {
return runSTM(makeSTM(cli, false, txQueue, so...), apply)
}
// NewSTM creates a new STM instance, using serializable snapshot isolation.
func NewSTM(cli *v3.Client, so ...STMOptionFunc) STM {
return makeSTM(cli, true, so...)
func NewSTM(cli *v3.Client, txQueue *commitQueue, so ...STMOptionFunc) STM {
return makeSTM(cli, true, txQueue, so...)
}
// makeSTM is the actual constructor of the stm. It first apply all passed
// options then creates the stm object and resets it before returning.
func makeSTM(cli *v3.Client, manual bool, so ...STMOptionFunc) *stm {
func makeSTM(cli *v3.Client, manual bool, txQueue *commitQueue,
so ...STMOptionFunc) *stm {
opts := &STMOptions{
ctx: cli.Ctx(),
}
@@ -215,6 +215,7 @@ func makeSTM(cli *v3.Client, manual bool, so ...STMOptionFunc) *stm {
s := &stm{
client: cli,
manual: manual,
txQueue: txQueue,
options: opts,
prefetch: make(map[string]stmGet),
}
@@ -230,50 +231,72 @@ func makeSTM(cli *v3.Client, manual bool, so ...STMOptionFunc) *stm {
// CommitError which is used to indicate a necessary retry.
func runSTM(s *stm, apply func(STM) error) error {
var (
retries int
stats CommitStats
err error
retries int
stats CommitStats
executeErr error
)
loop:
// In a loop try to apply and commit and roll back if the database has
// changed (CommitError).
for {
select {
// Check if the STM is aborted and break the retry loop if it is.
case <-s.options.ctx.Done():
err = fmt.Errorf("aborted")
break loop
done := make(chan struct{})
default:
execute := func() {
defer close(done)
for {
select {
// Check if the STM is aborted and break the retry loop
// if it is.
case <-s.options.ctx.Done():
executeErr = fmt.Errorf("aborted")
return
default:
}
stats, executeErr = s.commit()
// Re-apply only upon commit error (meaning the
// keys were changed).
if _, ok := executeErr.(CommitError); !ok {
// Anything that's not a CommitError
// aborts the transaction.
return
}
// Rollback before trying to re-apply.
s.Rollback()
retries++
// Re-apply the transaction closure.
if executeErr = apply(s); executeErr != nil {
return
}
}
// Apply the transaction closure and abort the STM if there was an
// application error.
if err = apply(s); err != nil {
break loop
}
stats, err = s.commit()
// Re-apply only upon commit error (meaning the database was changed).
if _, ok := err.(CommitError); !ok {
// Anything that's not a CommitError
// aborts the STM run loop.
break loop
}
// Rollback before trying to re-apply.
s.Rollback()
retries++
}
// Run the tx closure to construct the read and write sets.
// Also we expect that if there are no conflicting transactions
// in the queue, then we only run apply once.
if preApplyErr := apply(s); preApplyErr != nil {
return preApplyErr
}
// Queue up the transaction for execution.
s.txQueue.Add(execute, s.rset, s.wset)
// Wait for the transaction to execute, or break if aborted.
select {
case <-done:
case <-s.options.ctx.Done():
}
s.txQueue.Done(s.rset, s.wset)
if s.options.commitStatsCallback != nil {
stats.Retries = retries
s.options.commitStatsCallback(err == nil, stats)
s.options.commitStatsCallback(executeErr == nil, stats)
}
return err
return executeErr
}
// add inserts a txn response to the read set. This is useful when the txn
@@ -303,24 +326,14 @@ func (rs readSet) gets() []v3.Op {
return ops
}
// cmps returns a cmp list testing values in read set didn't change.
func (rs readSet) cmps(lset map[string]interface{}) []v3.Cmp {
if len(lset) > 0 {
cmps := make([]v3.Cmp, 0, len(lset))
for key := range lset {
if getValue, ok := rs[key]; ok {
cmps = append(
cmps,
v3.Compare(v3.ModRevision(key), "=", getValue.rev),
)
}
}
return cmps
}
// cmps returns a compare list which will serve as a precondition testing that
// the values in the read set didn't change.
func (rs readSet) cmps() []v3.Cmp {
cmps := make([]v3.Cmp, 0, len(rs))
for key, getValue := range rs {
cmps = append(cmps, v3.Compare(v3.ModRevision(key), "=", getValue.rev))
cmps = append(cmps, v3.Compare(
v3.ModRevision(key), "=", getValue.rev,
))
}
return cmps
@@ -370,6 +383,15 @@ func (s *stm) fetch(key string, opts ...v3.OpOption) ([]KV, error) {
}
}
if len(resp.Kvs) == 0 {
// Add assertion to the read set which will extend our commit
// constraint such that the commit will fail if the key is
// present in the database.
s.rset[key] = stmGet{
rev: 0,
}
}
var result []KV
// Fill the read set with key/values returned.
@@ -413,12 +435,22 @@ func (s *stm) Get(key string) ([]byte, error) {
// the prefetch set.
if getValue, ok := s.prefetch[key]; ok {
delete(s.prefetch, key)
s.rset[key] = getValue
// Use the prefetched value only if it is for
// an existing key.
if getValue.rev != 0 {
s.rset[key] = getValue
}
}
// Return value if alread in read set.
if getVal, ok := s.rset[key]; ok {
return []byte(getVal.val), nil
if getValue, ok := s.rset[key]; ok {
// Return the value if the rset contains an existing key.
if getValue.rev != 0 {
return []byte(getValue.val), nil
} else {
return nil, nil
}
}
// Fetch and return value.
@@ -435,13 +467,6 @@ func (s *stm) Get(key string) ([]byte, error) {
return nil, nil
}
// Lock adds a key to the lock set. If the lock set is
// not empty, we'll only check conflicts for the keys
// in the lock set.
func (s *stm) Lock(key string) {
s.lset[key] = nil
}
// First returns the first key/value matching prefix. If there's no key starting
// with prefix, Last will return nil.
func (s *stm) First(prefix string) (*KV, error) {
@@ -711,7 +736,7 @@ func (s *stm) OnCommit(cb func()) {
// because the keys have changed return a CommitError, otherwise return a
// DatabaseError.
func (s *stm) commit() (CommitStats, error) {
rset := s.rset.cmps(s.lset)
rset := s.rset.cmps()
wset := s.wset.cmps(s.revision + 1)
stats := CommitStats{
@@ -775,7 +800,6 @@ func (s *stm) Commit() error {
func (s *stm) Rollback() {
s.rset = make(map[string]stmGet)
s.wset = make(map[string]stmPut)
s.lset = make(map[string]interface{})
s.getOpts = nil
s.revision = math.MaxInt64 - 1
}
+110 -90
View File
@@ -6,7 +6,7 @@ import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func reverseKVs(a []KV) []KV {
@@ -21,27 +21,35 @@ func TestPutToEmpty(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
txQueue := NewCommitQueue(f.config.Ctx)
defer func() {
f.Cleanup()
txQueue.Wait()
}()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
apply := func(stm STM) error {
stm.Put("123", "abc")
return nil
}
err = RunSTM(db.cli, apply)
assert.NoError(t, err)
err = RunSTM(db.cli, apply, txQueue)
require.NoError(t, err)
assert.Equal(t, "abc", f.Get("123"))
require.Equal(t, "abc", f.Get("123"))
}
func TestGetPutDel(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.cleanup()
txQueue := NewCommitQueue(f.config.Ctx)
defer func() {
f.Cleanup()
txQueue.Wait()
}()
testKeyValues := []KV{
{"a", "1"},
@@ -56,71 +64,75 @@ func TestGetPutDel(t *testing.T) {
}
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
apply := func(stm STM) error {
// Get some non existing keys.
v, err := stm.Get("")
assert.NoError(t, err)
assert.Nil(t, v)
require.NoError(t, err)
require.Nil(t, v)
v, err = stm.Get("x")
assert.NoError(t, err)
assert.Nil(t, v)
require.NoError(t, err)
require.Nil(t, v)
// Get all existing keys.
for _, kv := range testKeyValues {
v, err = stm.Get(kv.key)
assert.NoError(t, err)
assert.Equal(t, []byte(kv.val), v)
require.NoError(t, err)
require.Equal(t, []byte(kv.val), v)
}
// Overwrite, then delete an existing key.
stm.Put("c", "6")
v, err = stm.Get("c")
assert.NoError(t, err)
assert.Equal(t, []byte("6"), v)
require.NoError(t, err)
require.Equal(t, []byte("6"), v)
stm.Del("c")
v, err = stm.Get("c")
assert.NoError(t, err)
assert.Nil(t, v)
require.NoError(t, err)
require.Nil(t, v)
// Re-add the deleted key.
stm.Put("c", "7")
v, err = stm.Get("c")
assert.NoError(t, err)
assert.Equal(t, []byte("7"), v)
require.NoError(t, err)
require.Equal(t, []byte("7"), v)
// Add a new key.
stm.Put("x", "x")
v, err = stm.Get("x")
assert.NoError(t, err)
assert.Equal(t, []byte("x"), v)
require.NoError(t, err)
require.Equal(t, []byte("x"), v)
return nil
}
err = RunSTM(db.cli, apply)
assert.NoError(t, err)
err = RunSTM(db.cli, apply, txQueue)
require.NoError(t, err)
assert.Equal(t, "1", f.Get("a"))
assert.Equal(t, "2", f.Get("b"))
assert.Equal(t, "7", f.Get("c"))
assert.Equal(t, "4", f.Get("d"))
assert.Equal(t, "5", f.Get("e"))
assert.Equal(t, "x", f.Get("x"))
require.Equal(t, "1", f.Get("a"))
require.Equal(t, "2", f.Get("b"))
require.Equal(t, "7", f.Get("c"))
require.Equal(t, "4", f.Get("d"))
require.Equal(t, "5", f.Get("e"))
require.Equal(t, "x", f.Get("x"))
}
func TestFirstLastNextPrev(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
txQueue := NewCommitQueue(f.config.Ctx)
defer func() {
f.Cleanup()
txQueue.Wait()
}()
testKeyValues := []KV{
{"kb", "1"},
@@ -134,44 +146,44 @@ func TestFirstLastNextPrev(t *testing.T) {
}
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
apply := func(stm STM) error {
// First/Last on valid multi item interval.
kv, err := stm.First("k")
assert.NoError(t, err)
assert.Equal(t, &KV{"kb", "1"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"kb", "1"}, kv)
kv, err = stm.Last("k")
assert.NoError(t, err)
assert.Equal(t, &KV{"ke", "4"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"ke", "4"}, kv)
// First/Last on single item interval.
kv, err = stm.First("w")
assert.NoError(t, err)
assert.Equal(t, &KV{"w", "w"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"w", "w"}, kv)
kv, err = stm.Last("w")
assert.NoError(t, err)
assert.Equal(t, &KV{"w", "w"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"w", "w"}, kv)
// Next/Prev on start/end.
kv, err = stm.Next("k", "ke")
assert.NoError(t, err)
assert.Nil(t, kv)
require.NoError(t, err)
require.Nil(t, kv)
kv, err = stm.Prev("k", "kb")
assert.NoError(t, err)
assert.Nil(t, kv)
require.NoError(t, err)
require.Nil(t, kv)
// Next/Prev in the middle.
kv, err = stm.Next("k", "kc")
assert.NoError(t, err)
assert.Equal(t, &KV{"kda", "3"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"kda", "3"}, kv)
kv, err = stm.Prev("k", "ke")
assert.NoError(t, err)
assert.Equal(t, &KV{"kda", "3"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"kda", "3"}, kv)
// Delete first item, then add an item before the
// deleted one. Check that First/Next will "jump"
@@ -180,12 +192,12 @@ func TestFirstLastNextPrev(t *testing.T) {
stm.Put("ka", "0")
kv, err = stm.First("k")
assert.NoError(t, err)
assert.Equal(t, &KV{"ka", "0"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"ka", "0"}, kv)
kv, err = stm.Prev("k", "kc")
assert.NoError(t, err)
assert.Equal(t, &KV{"ka", "0"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"ka", "0"}, kv)
// Similarly test that a new end is returned if
// the old end is deleted first.
@@ -193,19 +205,19 @@ func TestFirstLastNextPrev(t *testing.T) {
stm.Put("kf", "5")
kv, err = stm.Last("k")
assert.NoError(t, err)
assert.Equal(t, &KV{"kf", "5"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"kf", "5"}, kv)
kv, err = stm.Next("k", "kda")
assert.NoError(t, err)
assert.Equal(t, &KV{"kf", "5"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"kf", "5"}, kv)
// Overwrite one in the middle.
stm.Put("kda", "6")
kv, err = stm.Next("k", "kc")
assert.NoError(t, err)
assert.Equal(t, &KV{"kda", "6"}, kv)
require.NoError(t, err)
require.Equal(t, &KV{"kda", "6"}, kv)
// Add three in the middle, then delete one.
stm.Put("kdb", "7")
@@ -218,12 +230,12 @@ func TestFirstLastNextPrev(t *testing.T) {
var kvs []KV
curr, err := stm.First("k")
assert.NoError(t, err)
require.NoError(t, err)
for curr != nil {
kvs = append(kvs, *curr)
curr, err = stm.Next("k", curr.key)
assert.NoError(t, err)
require.NoError(t, err)
}
expected := []KV{
@@ -234,47 +246,51 @@ func TestFirstLastNextPrev(t *testing.T) {
{"kdd", "9"},
{"kf", "5"},
}
assert.Equal(t, expected, kvs)
require.Equal(t, expected, kvs)
// Similarly check that stepping from last to first
// returns the expected sequence.
kvs = []KV{}
curr, err = stm.Last("k")
assert.NoError(t, err)
require.NoError(t, err)
for curr != nil {
kvs = append(kvs, *curr)
curr, err = stm.Prev("k", curr.key)
assert.NoError(t, err)
require.NoError(t, err)
}
expected = reverseKVs(expected)
assert.Equal(t, expected, kvs)
require.Equal(t, expected, kvs)
return nil
}
err = RunSTM(db.cli, apply)
assert.NoError(t, err)
err = RunSTM(db.cli, apply, txQueue)
require.NoError(t, err)
assert.Equal(t, "0", f.Get("ka"))
assert.Equal(t, "2", f.Get("kc"))
assert.Equal(t, "6", f.Get("kda"))
assert.Equal(t, "7", f.Get("kdb"))
assert.Equal(t, "9", f.Get("kdd"))
assert.Equal(t, "5", f.Get("kf"))
assert.Equal(t, "w", f.Get("w"))
require.Equal(t, "0", f.Get("ka"))
require.Equal(t, "2", f.Get("kc"))
require.Equal(t, "6", f.Get("kda"))
require.Equal(t, "7", f.Get("kdb"))
require.Equal(t, "9", f.Get("kdd"))
require.Equal(t, "5", f.Get("kf"))
require.Equal(t, "w", f.Get("w"))
}
func TestCommitError(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
txQueue := NewCommitQueue(f.config.Ctx)
defer func() {
f.Cleanup()
txQueue.Wait()
}()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
// Preset DB state.
f.Put("123", "xyz")
@@ -285,10 +301,10 @@ func TestCommitError(t *testing.T) {
apply := func(stm STM) error {
// STM must have the key/value.
val, err := stm.Get("123")
assert.NoError(t, err)
require.NoError(t, err)
if cnt == 0 {
assert.Equal(t, []byte("xyz"), val)
require.Equal(t, []byte("xyz"), val)
// Put a conflicting key/value during the first apply.
f.Put("123", "def")
@@ -301,44 +317,48 @@ func TestCommitError(t *testing.T) {
return nil
}
err = RunSTM(db.cli, apply)
assert.NoError(t, err)
assert.Equal(t, 2, cnt)
err = RunSTM(db.cli, apply, txQueue)
require.NoError(t, err)
require.Equal(t, 2, cnt)
assert.Equal(t, "abc", f.Get("123"))
require.Equal(t, "abc", f.Get("123"))
}
func TestManualTxError(t *testing.T) {
t.Parallel()
f := NewEtcdTestFixture(t)
defer f.Cleanup()
txQueue := NewCommitQueue(f.config.Ctx)
defer func() {
f.Cleanup()
txQueue.Wait()
}()
db, err := newEtcdBackend(f.BackendConfig())
assert.NoError(t, err)
require.NoError(t, err)
// Preset DB state.
f.Put("123", "xyz")
stm := NewSTM(db.cli)
stm := NewSTM(db.cli, txQueue)
val, err := stm.Get("123")
assert.NoError(t, err)
assert.Equal(t, []byte("xyz"), val)
require.NoError(t, err)
require.Equal(t, []byte("xyz"), val)
// Put a conflicting key/value.
f.Put("123", "def")
// Should still get the original version.
val, err = stm.Get("123")
assert.NoError(t, err)
assert.Equal(t, []byte("xyz"), val)
require.NoError(t, err)
require.Equal(t, []byte("xyz"), val)
// Commit will fail with CommitError.
err = stm.Commit()
var e CommitError
assert.True(t, errors.As(err, &e))
require.True(t, errors.As(err, &e))
// We expect that the transacton indeed did not commit.
assert.Equal(t, "def", f.Get("123"))
require.Equal(t, "def", f.Get("123"))
}
+2 -2
View File
@@ -15,7 +15,7 @@ import (
func applyMigration(t *testing.T, beforeMigration, afterMigration func(d *DB),
migrationFunc migration, shouldFail bool, dryRun bool) {
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatal(err)
@@ -86,7 +86,7 @@ func applyMigration(t *testing.T, beforeMigration, afterMigration func(d *DB),
func TestVersionFetchPut(t *testing.T) {
t.Parallel()
db, cleanUp, err := makeTestDB()
db, cleanUp, err := MakeTestDB()
defer cleanUp()
if err != nil {
t.Fatal(err)
+2 -2
View File
@@ -13,7 +13,7 @@ import (
func TestLinkNodeEncodeDecode(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -110,7 +110,7 @@ func TestLinkNodeEncodeDecode(t *testing.T) {
func TestDeleteLinkNode(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
+7 -7
View File
@@ -56,7 +56,7 @@ func genInfo() (*PaymentCreationInfo, *HTLCAttemptInfo,
func TestPaymentControlSwitchFail(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
t.Fatalf("unable to init db: %v", err)
@@ -203,7 +203,7 @@ func TestPaymentControlSwitchFail(t *testing.T) {
func TestPaymentControlSwitchDoubleSend(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
@@ -286,7 +286,7 @@ func TestPaymentControlSwitchDoubleSend(t *testing.T) {
func TestPaymentControlSuccessesWithoutInFlight(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
@@ -319,7 +319,7 @@ func TestPaymentControlSuccessesWithoutInFlight(t *testing.T) {
func TestPaymentControlFailsWithoutInFlight(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
@@ -347,7 +347,7 @@ func TestPaymentControlFailsWithoutInFlight(t *testing.T) {
func TestPaymentControlDeleteNonInFligt(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
@@ -530,7 +530,7 @@ func TestPaymentControlMultiShard(t *testing.T) {
}
runSubTest := func(t *testing.T, test testCase) {
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
@@ -780,7 +780,7 @@ func TestPaymentControlMultiShard(t *testing.T) {
func TestPaymentControlMPPRecordValidation(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
defer cleanup()
if err != nil {
+2 -2
View File
@@ -399,7 +399,7 @@ func TestQueryPayments(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to init db: %v", err)
}
@@ -512,7 +512,7 @@ func TestQueryPayments(t *testing.T) {
// case where a specific duplicate is not found and the duplicates bucket is not
// present when we expect it to be.
func TestFetchPaymentWithSequenceNumber(t *testing.T) {
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
require.NoError(t, err)
defer cleanup()
+121
View File
@@ -0,0 +1,121 @@
package channeldb
import (
"bytes"
"errors"
"fmt"
"time"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/lightningnetwork/lnd/routing/route"
)
var (
// peersBucket is the name of a top level bucket in which we store
// information about our peers. Information for different peers is
// stored in buckets keyed by their public key.
//
//
// peers-bucket
// |
// |-- <peer-pubkey>
// | |--flap-count-key: <ts><flap count>
// |
// |-- <peer-pubkey>
// | |--flap-count-key: <ts><flap count>
peersBucket = []byte("peers-bucket")
// flapCountKey is a key used in the peer pubkey sub-bucket that stores
// the timestamp of a peer's last flap count and its all time flap
// count.
flapCountKey = []byte("flap-count")
)
var (
// ErrNoPeerBucket is returned when we try to read entries for a peer
// that is not tracked.
ErrNoPeerBucket = errors.New("peer bucket not found")
)
// FlapCount contains information about a peer's flap count.
type FlapCount struct {
// Count provides the total flap count for a peer.
Count uint32
// LastFlap is the timestamp of the last flap recorded for a peer.
LastFlap time.Time
}
// WriteFlapCounts writes the flap count for a set of peers to disk, creating a
// bucket for the peer's pubkey if necessary. Note that this function overwrites
// the current value.
func (d *DB) WriteFlapCounts(flapCounts map[route.Vertex]*FlapCount) error {
return d.Update(func(tx walletdb.ReadWriteTx) error {
// Run through our set of flap counts and record them for
// each peer, creating a bucket for the peer pubkey if required.
for peer, flapCount := range flapCounts {
peers := tx.ReadWriteBucket(peersBucket)
peerBucket, err := peers.CreateBucketIfNotExists(
peer[:],
)
if err != nil {
return err
}
var b bytes.Buffer
err = serializeTime(&b, flapCount.LastFlap)
if err != nil {
return err
}
if err = WriteElement(&b, flapCount.Count); err != nil {
return err
}
err = peerBucket.Put(flapCountKey, b.Bytes())
if err != nil {
return err
}
}
return nil
})
}
// ReadFlapCount attempts to read the flap count for a peer, failing if the
// peer is not found or we do not have flap count stored.
func (d *DB) ReadFlapCount(pubkey route.Vertex) (*FlapCount, error) {
var flapCount FlapCount
if err := d.View(func(tx walletdb.ReadTx) error {
peers := tx.ReadBucket(peersBucket)
peerBucket := peers.NestedReadBucket(pubkey[:])
if peerBucket == nil {
return ErrNoPeerBucket
}
flapBytes := peerBucket.Get(flapCountKey)
if flapBytes == nil {
return fmt.Errorf("flap count not recorded for: %v",
pubkey)
}
var (
err error
r = bytes.NewReader(flapBytes)
)
flapCount.LastFlap, err = deserializeTime(r)
if err != nil {
return err
}
return ReadElements(r, &flapCount.Count)
}); err != nil {
return nil, err
}
return &flapCount, nil
}
+50
View File
@@ -0,0 +1,50 @@
package channeldb
import (
"testing"
"time"
"github.com/lightningnetwork/lnd/routing/route"
"github.com/stretchr/testify/require"
)
// TestFlapCount tests lookup and writing of flap count to disk.
func TestFlapCount(t *testing.T) {
db, cleanup, err := MakeTestDB()
require.NoError(t, err)
defer cleanup()
// Try to read flap count for a peer that we have no records for.
_, err = db.ReadFlapCount(testPub)
require.Equal(t, ErrNoPeerBucket, err)
var (
testPub2 = route.Vertex{2, 2, 2}
peer1FlapCount = &FlapCount{
Count: 20,
LastFlap: time.Unix(100, 23),
}
peer2FlapCount = &FlapCount{
Count: 39,
LastFlap: time.Unix(200, 23),
}
)
peers := map[route.Vertex]*FlapCount{
testPub: peer1FlapCount,
testPub2: peer2FlapCount,
}
err = db.WriteFlapCounts(peers)
require.NoError(t, err)
// Lookup flap count for our first pubkey.
count, err := db.ReadFlapCount(testPub)
require.NoError(t, err)
require.Equal(t, peer1FlapCount, count)
// Lookup our flap count for the second peer.
count, err = db.ReadFlapCount(testPub2)
require.NoError(t, err)
require.Equal(t, peer2FlapCount, count)
}
+3 -3
View File
@@ -48,7 +48,7 @@ func TestPersistReport(t *testing.T) {
test := test
t.Run(test.name, func(t *testing.T) {
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
require.NoError(t, err)
defer cleanup()
@@ -85,7 +85,7 @@ func TestPersistReport(t *testing.T) {
// channel, testing that the appropriate error is returned based on the state
// of the existing bucket.
func TestFetchChannelReadBucket(t *testing.T) {
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
require.NoError(t, err)
defer cleanup()
@@ -197,7 +197,7 @@ func TestFetchChannelWriteBucket(t *testing.T) {
test := test
t.Run(test.name, func(t *testing.T) {
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
require.NoError(t, err)
defer cleanup()
+1 -1
View File
@@ -14,7 +14,7 @@ import (
func TestWaitingProofStore(t *testing.T) {
t.Parallel()
db, cleanup, err := makeTestDB()
db, cleanup, err := MakeTestDB()
if err != nil {
t.Fatalf("failed to make test database: %s", err)
}
+4 -4
View File
@@ -12,7 +12,7 @@ import (
func TestWitnessCacheSha256Retrieval(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -57,7 +57,7 @@ func TestWitnessCacheSha256Retrieval(t *testing.T) {
func TestWitnessCacheSha256Deletion(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -108,7 +108,7 @@ func TestWitnessCacheSha256Deletion(t *testing.T) {
func TestWitnessCacheUnknownWitness(t *testing.T) {
t.Parallel()
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
@@ -127,7 +127,7 @@ func TestWitnessCacheUnknownWitness(t *testing.T) {
// TestAddSha256Witnesses tests that insertion using AddSha256Witnesses behaves
// identically to the insertion via the generalized interface.
func TestAddSha256Witnesses(t *testing.T) {
cdb, cleanUp, err := makeTestDB()
cdb, cleanUp, err := MakeTestDB()
if err != nil {
t.Fatalf("unable to make test database: %v", err)
}
+1 -1
View File
@@ -271,7 +271,7 @@ func (s *server) ConnectPeer(nodePub *btcec.PublicKey, addrs []net.Addr) error {
// Attempt to connect to the peer using this full address. If
// we're unable to connect to them, then we'll try the next
// address in place of it.
err := s.ConnectToPeer(netAddr, true)
err := s.ConnectToPeer(netAddr, true, s.cfg.ConnectionTimeout)
// If we're already connected to this peer, then we don't
// consider this an error, so we'll exit here.
-182
View File
@@ -1,182 +0,0 @@
package main
import (
"context"
"encoding/hex"
"fmt"
"io/ioutil"
"net"
"strings"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/macaroons"
"github.com/urfave/cli"
"gopkg.in/macaroon.v2"
)
var bakeMacaroonCommand = cli.Command{
Name: "bakemacaroon",
Category: "Macaroons",
Usage: "Bakes a new macaroon with the provided list of permissions " +
"and restrictions",
ArgsUsage: "[--save_to=] [--timeout=] [--ip_address=] permissions...",
Description: `
Bake a new macaroon that grants the provided permissions and
optionally adds restrictions (timeout, IP address) to it.
The new macaroon can either be shown on command line in hex serialized
format or it can be saved directly to a file using the --save_to
argument.
A permission is a tuple of an entity and an action, separated by a
colon. Multiple operations can be added as arguments, for example:
lncli bakemacaroon info:read invoices:write foo:bar
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "save_to",
Usage: "save the created macaroon to this file " +
"using the default binary format",
},
cli.Uint64Flag{
Name: "timeout",
Usage: "the number of seconds the macaroon will be " +
"valid before it times out",
},
cli.StringFlag{
Name: "ip_address",
Usage: "the IP address the macaroon will be bound to",
},
},
Action: actionDecorator(bakeMacaroon),
}
func bakeMacaroon(ctx *cli.Context) error {
client, cleanUp := getClient(ctx)
defer cleanUp()
// Show command help if no arguments.
if ctx.NArg() == 0 {
return cli.ShowCommandHelp(ctx, "bakemacaroon")
}
args := ctx.Args()
var (
savePath string
timeout int64
ipAddress net.IP
parsedPermissions []*lnrpc.MacaroonPermission
err error
)
if ctx.String("save_to") != "" {
savePath = cleanAndExpandPath(ctx.String("save_to"))
}
if ctx.IsSet("timeout") {
timeout = ctx.Int64("timeout")
if timeout <= 0 {
return fmt.Errorf("timeout must be greater than 0")
}
}
if ctx.IsSet("ip_address") {
ipAddress = net.ParseIP(ctx.String("ip_address"))
if ipAddress == nil {
return fmt.Errorf("unable to parse ip_address: %s",
ctx.String("ip_address"))
}
}
// A command line argument can't be an empty string. So we'll check each
// entry if it's a valid entity:action tuple. The content itself is
// validated server side. We just make sure we can parse it correctly.
for _, permission := range args {
tuple := strings.Split(permission, ":")
if len(tuple) != 2 {
return fmt.Errorf("unable to parse "+
"permission tuple: %s", permission)
}
entity, action := tuple[0], tuple[1]
if entity == "" {
return fmt.Errorf("invalid permission [%s]. entity "+
"cannot be empty", permission)
}
if action == "" {
return fmt.Errorf("invalid permission [%s]. action "+
"cannot be empty", permission)
}
// No we can assume that we have a formally valid entity:action
// tuple. The rest of the validation happens server side.
parsedPermissions = append(
parsedPermissions, &lnrpc.MacaroonPermission{
Entity: entity,
Action: action,
},
)
}
// Now we have gathered all the input we need and can do the actual
// RPC call.
req := &lnrpc.BakeMacaroonRequest{
Permissions: parsedPermissions,
}
resp, err := client.BakeMacaroon(context.Background(), req)
if err != nil {
return err
}
// Now we should have gotten a valid macaroon. Unmarshal it so we can
// add first-party caveats (if necessary) to it.
macBytes, err := hex.DecodeString(resp.Macaroon)
if err != nil {
return err
}
unmarshalMac := &macaroon.Macaroon{}
if err = unmarshalMac.UnmarshalBinary(macBytes); err != nil {
return err
}
// Now apply the desired constraints to the macaroon. This will always
// create a new macaroon object, even if no constraints are added.
macConstraints := make([]macaroons.Constraint, 0)
if timeout > 0 {
macConstraints = append(
macConstraints, macaroons.TimeoutConstraint(timeout),
)
}
if ipAddress != nil {
macConstraints = append(
macConstraints,
macaroons.IPLockConstraint(ipAddress.String()),
)
}
constrainedMac, err := macaroons.AddConstraints(
unmarshalMac, macConstraints...,
)
if err != nil {
return err
}
macBytes, err = constrainedMac.MarshalBinary()
if err != nil {
return err
}
// Now we can output the result. We either write it binary serialized to
// a file or write to the standard output using hex encoding.
switch {
case savePath != "":
err = ioutil.WriteFile(savePath, macBytes, 0644)
if err != nil {
return err
}
fmt.Printf("Macaroon saved to %s\n", savePath)
default:
fmt.Printf("%s\n", hex.EncodeToString(macBytes))
}
return nil
}
+413
View File
@@ -0,0 +1,413 @@
package main
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"io/ioutil"
"net"
"strconv"
"strings"
"github.com/golang/protobuf/proto"
"github.com/lightningnetwork/lnd/lncfg"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/macaroons"
"github.com/urfave/cli"
"gopkg.in/macaroon-bakery.v2/bakery"
"gopkg.in/macaroon.v2"
)
var bakeMacaroonCommand = cli.Command{
Name: "bakemacaroon",
Category: "Macaroons",
Usage: "Bakes a new macaroon with the provided list of permissions " +
"and restrictions.",
ArgsUsage: "[--save_to=] [--timeout=] [--ip_address=] permissions...",
Description: `
Bake a new macaroon that grants the provided permissions and
optionally adds restrictions (timeout, IP address) to it.
The new macaroon can either be shown on command line in hex serialized
format or it can be saved directly to a file using the --save_to
argument.
A permission is a tuple of an entity and an action, separated by a
colon. Multiple operations can be added as arguments, for example:
lncli bakemacaroon info:read invoices:write foo:bar
For even more fine-grained permission control, it is also possible to
specify single RPC method URIs that are allowed to be accessed by a
macaroon. This can be achieved by specifying "uri:<methodURI>" pairs,
for example:
lncli bakemacaroon uri:/lnrpc.Lightning/GetInfo uri:/verrpc.Versioner/GetVersion
The macaroon created by this command would only be allowed to use the
"lncli getinfo" and "lncli version" commands.
To get a list of all available URIs and permissions, use the
"lncli listpermissions" command.
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "save_to",
Usage: "save the created macaroon to this file " +
"using the default binary format",
},
cli.Uint64Flag{
Name: "timeout",
Usage: "the number of seconds the macaroon will be " +
"valid before it times out",
},
cli.StringFlag{
Name: "ip_address",
Usage: "the IP address the macaroon will be bound to",
},
cli.Uint64Flag{
Name: "root_key_id",
Usage: "the numerical root key ID used to create the macaroon",
},
},
Action: actionDecorator(bakeMacaroon),
}
func bakeMacaroon(ctx *cli.Context) error {
client, cleanUp := getClient(ctx)
defer cleanUp()
// Show command help if no arguments.
if ctx.NArg() == 0 {
return cli.ShowCommandHelp(ctx, "bakemacaroon")
}
args := ctx.Args()
var (
savePath string
timeout int64
ipAddress net.IP
rootKeyID uint64
parsedPermissions []*lnrpc.MacaroonPermission
err error
)
if ctx.String("save_to") != "" {
savePath = lncfg.CleanAndExpandPath(ctx.String("save_to"))
}
if ctx.IsSet("timeout") {
timeout = ctx.Int64("timeout")
if timeout <= 0 {
return fmt.Errorf("timeout must be greater than 0")
}
}
if ctx.IsSet("ip_address") {
ipAddress = net.ParseIP(ctx.String("ip_address"))
if ipAddress == nil {
return fmt.Errorf("unable to parse ip_address: %s",
ctx.String("ip_address"))
}
}
if ctx.IsSet("root_key_id") {
rootKeyID = ctx.Uint64("root_key_id")
}
// A command line argument can't be an empty string. So we'll check each
// entry if it's a valid entity:action tuple. The content itself is
// validated server side. We just make sure we can parse it correctly.
for _, permission := range args {
tuple := strings.Split(permission, ":")
if len(tuple) != 2 {
return fmt.Errorf("unable to parse "+
"permission tuple: %s", permission)
}
entity, action := tuple[0], tuple[1]
if entity == "" {
return fmt.Errorf("invalid permission [%s]. entity "+
"cannot be empty", permission)
}
if action == "" {
return fmt.Errorf("invalid permission [%s]. action "+
"cannot be empty", permission)
}
// No we can assume that we have a formally valid entity:action
// tuple. The rest of the validation happens server side.
parsedPermissions = append(
parsedPermissions, &lnrpc.MacaroonPermission{
Entity: entity,
Action: action,
},
)
}
// Now we have gathered all the input we need and can do the actual
// RPC call.
req := &lnrpc.BakeMacaroonRequest{
Permissions: parsedPermissions,
RootKeyId: rootKeyID,
}
resp, err := client.BakeMacaroon(context.Background(), req)
if err != nil {
return err
}
// Now we should have gotten a valid macaroon. Unmarshal it so we can
// add first-party caveats (if necessary) to it.
macBytes, err := hex.DecodeString(resp.Macaroon)
if err != nil {
return err
}
unmarshalMac := &macaroon.Macaroon{}
if err = unmarshalMac.UnmarshalBinary(macBytes); err != nil {
return err
}
// Now apply the desired constraints to the macaroon. This will always
// create a new macaroon object, even if no constraints are added.
macConstraints := make([]macaroons.Constraint, 0)
if timeout > 0 {
macConstraints = append(
macConstraints, macaroons.TimeoutConstraint(timeout),
)
}
if ipAddress != nil {
macConstraints = append(
macConstraints,
macaroons.IPLockConstraint(ipAddress.String()),
)
}
constrainedMac, err := macaroons.AddConstraints(
unmarshalMac, macConstraints...,
)
if err != nil {
return err
}
macBytes, err = constrainedMac.MarshalBinary()
if err != nil {
return err
}
// Now we can output the result. We either write it binary serialized to
// a file or write to the standard output using hex encoding.
switch {
case savePath != "":
err = ioutil.WriteFile(savePath, macBytes, 0644)
if err != nil {
return err
}
fmt.Printf("Macaroon saved to %s\n", savePath)
default:
fmt.Printf("%s\n", hex.EncodeToString(macBytes))
}
return nil
}
var listMacaroonIDsCommand = cli.Command{
Name: "listmacaroonids",
Category: "Macaroons",
Usage: "List all macaroons root key IDs in use.",
Action: actionDecorator(listMacaroonIDs),
}
func listMacaroonIDs(ctx *cli.Context) error {
client, cleanUp := getClient(ctx)
defer cleanUp()
req := &lnrpc.ListMacaroonIDsRequest{}
resp, err := client.ListMacaroonIDs(context.Background(), req)
if err != nil {
return err
}
printRespJSON(resp)
return nil
}
var deleteMacaroonIDCommand = cli.Command{
Name: "deletemacaroonid",
Category: "Macaroons",
Usage: "Delete a specific macaroon ID.",
ArgsUsage: "root_key_id",
Description: `
Remove a macaroon ID using the specified root key ID. For example:
lncli deletemacaroonid 1
WARNING
When the ID is deleted, all macaroons created from that root key will
be invalidated.
Note that the default root key ID 0 cannot be deleted.
`,
Action: actionDecorator(deleteMacaroonID),
}
func deleteMacaroonID(ctx *cli.Context) error {
client, cleanUp := getClient(ctx)
defer cleanUp()
// Validate args length. Only one argument is allowed.
if ctx.NArg() != 1 {
return cli.ShowCommandHelp(ctx, "deletemacaroonid")
}
rootKeyIDString := ctx.Args().First()
// Convert string into uint64.
rootKeyID, err := strconv.ParseUint(rootKeyIDString, 10, 64)
if err != nil {
return fmt.Errorf("root key ID must be a positive integer")
}
// Check that the value is not equal to DefaultRootKeyID. Note that the
// server also validates the root key ID when removing it. However, we check
// it here too so that we can give users a nice warning.
if bytes.Equal([]byte(rootKeyIDString), macaroons.DefaultRootKeyID) {
return fmt.Errorf("deleting the default root key ID 0 is not allowed")
}
// Make the actual RPC call.
req := &lnrpc.DeleteMacaroonIDRequest{
RootKeyId: rootKeyID,
}
resp, err := client.DeleteMacaroonID(context.Background(), req)
if err != nil {
return err
}
printRespJSON(resp)
return nil
}
var listPermissionsCommand = cli.Command{
Name: "listpermissions",
Category: "Macaroons",
Usage: "Lists all RPC method URIs and the macaroon permissions they " +
"require to be invoked.",
Action: actionDecorator(listPermissions),
}
func listPermissions(ctx *cli.Context) error {
client, cleanUp := getClient(ctx)
defer cleanUp()
request := &lnrpc.ListPermissionsRequest{}
response, err := client.ListPermissions(context.Background(), request)
if err != nil {
return err
}
printRespJSON(response)
return nil
}
type macaroonContent struct {
Version uint16 `json:"version"`
Location string `json:"location"`
RootKeyID string `json:"root_key_id"`
Permissions []string `json:"permissions"`
Caveats []string `json:"caveats"`
}
var printMacaroonCommand = cli.Command{
Name: "printmacaroon",
Category: "Macaroons",
Usage: "Print the content of a macaroon in a human readable format.",
ArgsUsage: "[macaroon_content_hex]",
Description: `
Decode a macaroon and show its content in a more human readable format.
The macaroon can either be passed as a hex encoded positional parameter
or loaded from a file.
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "macaroon_file",
Usage: "load the macaroon from a file instead of the " +
"command line directly",
},
},
Action: actionDecorator(printMacaroon),
}
func printMacaroon(ctx *cli.Context) error {
// Show command help if no arguments or flags are set.
if ctx.NArg() == 0 && ctx.NumFlags() == 0 {
return cli.ShowCommandHelp(ctx, "printmacaroon")
}
var (
macBytes []byte
err error
args = ctx.Args()
)
switch {
case ctx.IsSet("macaroon_file"):
macPath := lncfg.CleanAndExpandPath(ctx.String("macaroon_file"))
// Load the specified macaroon file.
macBytes, err = ioutil.ReadFile(macPath)
if err != nil {
return fmt.Errorf("unable to read macaroon path %v: %v",
macPath, err)
}
case args.Present():
macBytes, err = hex.DecodeString(args.First())
if err != nil {
return fmt.Errorf("unable to hex decode macaroon: %v",
err)
}
default:
return fmt.Errorf("macaroon parameter missing")
}
// Decode the macaroon and its protobuf encoded internal identifier.
mac := &macaroon.Macaroon{}
if err = mac.UnmarshalBinary(macBytes); err != nil {
return fmt.Errorf("unable to decode macaroon: %v", err)
}
rawID := mac.Id()
if rawID[0] != byte(bakery.LatestVersion) {
return fmt.Errorf("invalid macaroon version: %x", rawID)
}
decodedID := &lnrpc.MacaroonId{}
idProto := rawID[1:]
err = proto.Unmarshal(idProto, decodedID)
if err != nil {
return fmt.Errorf("unable to decode macaroon version: %v", err)
}
// Prepare everything to be printed in a more human readable format.
content := &macaroonContent{
Version: uint16(mac.Version()),
Location: mac.Location(),
RootKeyID: string(decodedID.StorageId),
Permissions: nil,
Caveats: nil,
}
for _, caveat := range mac.Caveats() {
content.Caveats = append(content.Caveats, string(caveat.Id))
}
for _, op := range decodedID.Ops {
for _, action := range op.Actions {
permission := fmt.Sprintf("%s:%s", op.Entity, action)
content.Permissions = append(
content.Permissions, permission,
)
}
}
printJSON(content)
return nil
}
+55 -18
View File
@@ -1,6 +1,7 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
@@ -11,6 +12,7 @@ import (
"strings"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/lnwallet/chanfunding"
@@ -43,9 +45,9 @@ Base64 encoded PSBT: `
userMsgSign = `
PSBT verified by lnd, please continue the funding flow by signing the PSBT by
all required parties/devices. Once the transaction is fully signed, paste it
again here.
again here either in base64 PSBT or hex encoded raw wire TX format.
Base64 encoded signed PSBT: `
Signed base64 encoded PSBT or hex encoded raw wire TX: `
)
// TODO(roasbeef): change default number of confirmations
@@ -414,7 +416,10 @@ func openChannelPsbt(ctx *cli.Context, client lnrpc.LightningClient,
if err != nil {
return fmt.Errorf("opening stream to server failed: %v", err)
}
signal.Intercept()
if err := signal.Intercept(); err != nil {
return err
}
// We also need to spawn a goroutine that reads from the server. This
// will copy the messages to the channel as long as they come in or add
@@ -502,7 +507,7 @@ func openChannelPsbt(ctx *cli.Context, client lnrpc.LightningClient,
return fmt.Errorf("reading from console "+
"failed: %v", err)
}
psbt, err := base64.StdEncoding.DecodeString(
fundedPsbt, err := base64.StdEncoding.DecodeString(
strings.TrimSpace(psbtBase64),
)
if err != nil {
@@ -512,7 +517,7 @@ func openChannelPsbt(ctx *cli.Context, client lnrpc.LightningClient,
verifyMsg := &lnrpc.FundingTransitionMsg{
Trigger: &lnrpc.FundingTransitionMsg_PsbtVerify{
PsbtVerify: &lnrpc.FundingPsbtVerify{
FundedPsbt: psbt,
FundedPsbt: fundedPsbt,
PendingChanId: pendingChanID[:],
},
},
@@ -528,7 +533,7 @@ func openChannelPsbt(ctx *cli.Context, client lnrpc.LightningClient,
fmt.Print(userMsgSign)
// Read the signed PSBT and send it to lnd.
psbtBase64, err = readLine(quit)
finalTxStr, err := readLine(quit)
if err == io.EOF {
return nil
}
@@ -536,22 +541,16 @@ func openChannelPsbt(ctx *cli.Context, client lnrpc.LightningClient,
return fmt.Errorf("reading from console "+
"failed: %v", err)
}
psbt, err = base64.StdEncoding.DecodeString(
strings.TrimSpace(psbtBase64),
finalizeMsg, err := finalizeMsgFromString(
finalTxStr, pendingChanID[:],
)
if err != nil {
return fmt.Errorf("base64 decode failed: %v",
err)
return err
}
finalizeMsg := &lnrpc.FundingTransitionMsg{
Trigger: &lnrpc.FundingTransitionMsg_PsbtFinalize{
PsbtFinalize: &lnrpc.FundingPsbtFinalize{
SignedPsbt: psbt,
PendingChanId: pendingChanID[:],
},
},
transitionMsg := &lnrpc.FundingTransitionMsg{
Trigger: finalizeMsg,
}
err = sendFundingState(ctxc, ctx, finalizeMsg)
err = sendFundingState(ctxc, ctx, transitionMsg)
if err != nil {
return fmt.Errorf("finalizing PSBT funding "+
"flow failed: %v", err)
@@ -683,3 +682,41 @@ func sendFundingState(cancelCtx context.Context, cliCtx *cli.Context,
_, err := client.FundingStateStep(cancelCtx, msg)
return err
}
// finalizeMsgFromString creates the final message for the PsbtFinalize step
// from either a hex encoded raw wire transaction or a base64 encoded PSBT
// packet.
func finalizeMsgFromString(tx string,
pendingChanID []byte) (*lnrpc.FundingTransitionMsg_PsbtFinalize, error) {
rawTx, err := hex.DecodeString(strings.TrimSpace(tx))
if err == nil {
// Hex decoding succeeded so we assume we have a raw wire format
// transaction. Let's submit that instead of a PSBT packet.
tx := &wire.MsgTx{}
err := tx.Deserialize(bytes.NewReader(rawTx))
if err != nil {
return nil, fmt.Errorf("deserializing as raw wire "+
"transaction failed: %v", err)
}
return &lnrpc.FundingTransitionMsg_PsbtFinalize{
PsbtFinalize: &lnrpc.FundingPsbtFinalize{
FinalRawTx: rawTx,
PendingChanId: pendingChanID,
},
}, nil
}
// If the string isn't a hex encoded transaction, we assume it must be
// a base64 encoded PSBT packet.
psbtBytes, err := base64.StdEncoding.DecodeString(strings.TrimSpace(tx))
if err != nil {
return nil, fmt.Errorf("base64 decode failed: %v", err)
}
return &lnrpc.FundingTransitionMsg_PsbtFinalize{
PsbtFinalize: &lnrpc.FundingPsbtFinalize{
SignedPsbt: psbtBytes,
PendingChanId: pendingChanID,
},
}, nil
}
+449
View File
@@ -0,0 +1,449 @@
package main
import (
"fmt"
"io/ioutil"
"os"
"path"
"strings"
"github.com/btcsuite/btcutil"
"github.com/lightningnetwork/lnd/lncfg"
"github.com/urfave/cli"
"gopkg.in/macaroon.v2"
)
var (
// defaultLncliDir is the default directory to store the profile file
// in. This defaults to:
// C:\Users\<username>\AppData\Local\Lncli\ on Windows
// ~/.lncli/ on Linux
// ~/Library/Application Support/Lncli/ on MacOS
defaultLncliDir = btcutil.AppDataDir("lncli", false)
// defaultProfileFile is the full, absolute path of the profile file.
defaultProfileFile = path.Join(defaultLncliDir, "profiles.json")
)
var profileSubCommand = cli.Command{
Name: "profile",
Category: "Profiles",
Usage: "Create and manage lncli profiles",
Description: `
Profiles for lncli are an easy and comfortable way to manage multiple
nodes from the command line by storing node specific parameters like RPC
host, network, TLS certificate path or macaroons in a named profile.
To use a predefined profile, just use the '--profile=myprofile' (or
short version '-p=myprofile') with any lncli command.
A default profile can also be defined, lncli will then always use the
connection/node parameters from that profile instead of the default
values.
WARNING: Setting a default profile changes the default behavior of
lncli! To disable the use of the default profile for a single command,
set '--profile= '.
The profiles are stored in a file called profiles.json in the user's
home directory, for example:
C:\Users\<username>\AppData\Local\Lncli\profiles.json on Windows
~/.lncli/profiles.json on Linux
~/Library/Application Support/Lncli/profiles.json on MacOS
`,
Subcommands: []cli.Command{
profileListCommand,
profileAddCommand,
profileRemoveCommand,
profileSetDefaultCommand,
profileUnsetDefaultCommand,
profileAddMacaroonCommand,
},
}
var profileListCommand = cli.Command{
Name: "list",
Usage: "Lists all lncli profiles",
Action: profileList,
}
func profileList(_ *cli.Context) error {
f, err := loadProfileFile(defaultProfileFile)
if err != nil {
return err
}
printJSON(f)
return nil
}
var profileAddCommand = cli.Command{
Name: "add",
Usage: "Add a new profile",
ArgsUsage: "name",
Description: `
Add a new named profile to the main profiles.json. All global options
(see 'lncli --help') passed into this command are stored in that named
profile.
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "name",
Usage: "the name of the new profile",
},
cli.BoolFlag{
Name: "default",
Usage: "set the new profile to be the default profile",
},
},
Action: profileAdd,
}
func profileAdd(ctx *cli.Context) error {
if ctx.NArg() == 0 && ctx.NumFlags() == 0 {
return cli.ShowCommandHelp(ctx, "add")
}
// Load the default profile file or create a new one if it doesn't exist
// yet.
f, err := loadProfileFile(defaultProfileFile)
switch {
case err == errNoProfileFile:
f = &profileFile{}
_ = os.MkdirAll(path.Dir(defaultProfileFile), 0700)
case err != nil:
return err
}
// Create a profile struct from all the global options.
profile, err := profileFromContext(ctx, true)
if err != nil {
return fmt.Errorf("could not load global options: %v", err)
}
// Finally, all that's left is to get the profile name from either
// positional argument or flag.
args := ctx.Args()
switch {
case ctx.IsSet("name"):
profile.Name = ctx.String("name")
case args.Present():
profile.Name = args.First()
default:
return fmt.Errorf("name argument missing")
}
// Is there already a profile with that name?
for _, p := range f.Profiles {
if p.Name == profile.Name {
return fmt.Errorf("a profile with the name %s already "+
"exists", profile.Name)
}
}
// Do we need to update the default entry to be this one?
if ctx.Bool("default") {
f.Default = profile.Name
}
// All done, store the updated profile file.
f.Profiles = append(f.Profiles, profile)
if err = saveProfileFile(defaultProfileFile, f); err != nil {
return fmt.Errorf("error writing profile file %s: %v",
defaultProfileFile, err)
}
fmt.Printf("Profile %s added to file %s.\n", profile.Name,
defaultProfileFile)
return nil
}
var profileRemoveCommand = cli.Command{
Name: "remove",
Usage: "Remove a profile",
ArgsUsage: "name",
Description: `Remove the specified profile from the profile file.`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "name",
Usage: "the name of the profile to delete",
},
},
Action: profileRemove,
}
func profileRemove(ctx *cli.Context) error {
if ctx.NArg() == 0 && ctx.NumFlags() == 0 {
return cli.ShowCommandHelp(ctx, "remove")
}
// Load the default profile file.
f, err := loadProfileFile(defaultProfileFile)
if err != nil {
return fmt.Errorf("could not load profile file: %v", err)
}
// Get the profile name from either positional argument or flag.
var (
args = ctx.Args()
name string
found = false
)
switch {
case ctx.IsSet("name"):
name = ctx.String("name")
case args.Present():
name = args.First()
default:
return fmt.Errorf("name argument missing")
}
// Create a copy of all profiles but don't include the one to delete.
newProfiles := make([]*profileEntry, 0, len(f.Profiles)-1)
for _, p := range f.Profiles {
// Skip the one we want to delete.
if p.Name == name {
found = true
if p.Name == f.Default {
fmt.Println("Warning: removing default profile.")
}
continue
}
// Keep all others.
newProfiles = append(newProfiles, p)
}
// If what we were looking for didn't exist in the first place, there's
// no need for updating the file.
if !found {
return fmt.Errorf("profile with name %s not found in file",
name)
}
// Great, everything updated, now let's save the file.
f.Profiles = newProfiles
return saveProfileFile(defaultProfileFile, f)
}
var profileSetDefaultCommand = cli.Command{
Name: "setdefault",
Usage: "Set the default profile",
ArgsUsage: "name",
Description: `
Set a specified profile to be used as the default profile.
WARNING: Setting a default profile changes the default behavior of
lncli! To disable the use of the default profile for a single command,
set '--profile= '.
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "name",
Usage: "the name of the profile to set as default",
},
},
Action: profileSetDefault,
}
func profileSetDefault(ctx *cli.Context) error {
if ctx.NArg() == 0 && ctx.NumFlags() == 0 {
return cli.ShowCommandHelp(ctx, "setdefault")
}
// Load the default profile file.
f, err := loadProfileFile(defaultProfileFile)
if err != nil {
return fmt.Errorf("could not load profile file: %v", err)
}
// Get the profile name from either positional argument or flag.
var (
args = ctx.Args()
name string
found = false
)
switch {
case ctx.IsSet("name"):
name = ctx.String("name")
case args.Present():
name = args.First()
default:
return fmt.Errorf("name argument missing")
}
// Make sure the new default profile actually exists.
for _, p := range f.Profiles {
if p.Name == name {
found = true
f.Default = p.Name
break
}
}
// If the default profile doesn't exist, there's no need for updating
// the file.
if !found {
return fmt.Errorf("profile with name %s not found in file",
name)
}
// Great, everything updated, now let's save the file.
return saveProfileFile(defaultProfileFile, f)
}
var profileUnsetDefaultCommand = cli.Command{
Name: "unsetdefault",
Usage: "Unsets the default profile",
Description: `
Disables the use of a default profile and restores lncli to its original
behavior.
`,
Action: profileUnsetDefault,
}
func profileUnsetDefault(_ *cli.Context) error {
// Load the default profile file.
f, err := loadProfileFile(defaultProfileFile)
if err != nil {
return fmt.Errorf("could not load profile file: %v", err)
}
// Save the file with the flag disabled.
f.Default = ""
return saveProfileFile(defaultProfileFile, f)
}
var profileAddMacaroonCommand = cli.Command{
Name: "addmacaroon",
Usage: "Add a macaroon to a profile's macaroon jar",
ArgsUsage: "macaroon-name",
Description: `
Add an additional macaroon specified by the global option --macaroonpath
to an existing profile's macaroon jar.
If no profile is selected, the macaroon is added to the default profile
(if one exists). To add a macaroon to a specific profile, use the global
--profile=myprofile option.
If multiple macaroons exist in a profile's macaroon jar, the one to use
can be specified with the global option --macfromjar=xyz.
`,
Flags: []cli.Flag{
cli.StringFlag{
Name: "name",
Usage: "the name of the macaroon",
},
cli.BoolFlag{
Name: "default",
Usage: "set the new macaroon to be the default " +
"macaroon in the jar",
},
},
Action: profileAddMacaroon,
}
func profileAddMacaroon(ctx *cli.Context) error {
if ctx.NArg() == 0 && ctx.NumFlags() == 0 {
return cli.ShowCommandHelp(ctx, "addmacaroon")
}
// Load the default profile file or create a new one if it doesn't exist
// yet.
f, err := loadProfileFile(defaultProfileFile)
if err != nil {
return fmt.Errorf("could not load profile file: %v", err)
}
// Finally, all that's left is to get the profile name from either
// positional argument or flag.
var (
args = ctx.Args()
profileName string
macName string
)
switch {
case ctx.IsSet("name"):
macName = ctx.String("name")
case args.Present():
macName = args.First()
default:
return fmt.Errorf("name argument missing")
}
// Make sure the user actually set a macaroon path to use.
if !ctx.GlobalIsSet("macaroonpath") {
return fmt.Errorf("macaroonpath global option missing")
}
// Find out which profile we should add the macaroon. The global flag
// takes precedence over the default profile.
if f.Default != "" {
profileName = f.Default
}
if ctx.GlobalIsSet("profile") {
profileName = ctx.GlobalString("profile")
}
if len(strings.TrimSpace(profileName)) == 0 {
return fmt.Errorf("no profile specified and no default " +
"profile exists")
}
// Is there a profile with that name?
var selectedProfile *profileEntry
for _, p := range f.Profiles {
if p.Name == profileName {
selectedProfile = p
break
}
}
if selectedProfile == nil {
return fmt.Errorf("profile with name %s not found", profileName)
}
// Does a macaroon with that name already exist?
for _, m := range selectedProfile.Macaroons.Jar {
if m.Name == macName {
return fmt.Errorf("a macaroon with the name %s "+
"already exists", macName)
}
}
// Do we need to update the default entry to be this one?
if ctx.Bool("default") {
selectedProfile.Macaroons.Default = macName
}
// Now load and possibly encrypt the macaroon file.
macPath := lncfg.CleanAndExpandPath(ctx.GlobalString("macaroonpath"))
macBytes, err := ioutil.ReadFile(macPath)
if err != nil {
return fmt.Errorf("unable to read macaroon path: %v", err)
}
mac := &macaroon.Macaroon{}
if err = mac.UnmarshalBinary(macBytes); err != nil {
return fmt.Errorf("unable to decode macaroon: %v", err)
}
macEntry := &macaroonEntry{
Name: macName,
}
if err = macEntry.storeMacaroon(mac, nil); err != nil {
return fmt.Errorf("unable to store macaroon: %v", err)
}
// All done, store the updated profile file.
selectedProfile.Macaroons.Jar = append(
selectedProfile.Macaroons.Jar, macEntry,
)
if err = saveProfileFile(defaultProfileFile, f); err != nil {
return fmt.Errorf("error writing profile file %s: %v",
defaultProfileFile, err)
}
fmt.Printf("Macaroon %s added to profile %s in file %s.\n", macName,
selectedProfile.Name, defaultProfileFile)
return nil
}
+35 -36
View File
@@ -14,7 +14,6 @@ import (
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/btcsuite/btcd/chaincfg/chainhash"
@@ -26,7 +25,6 @@ import (
"github.com/lightningnetwork/lnd/routing/route"
"github.com/lightningnetwork/lnd/walletunlocker"
"github.com/urfave/cli"
"golang.org/x/crypto/ssh/terminal"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
@@ -221,7 +219,7 @@ var sendCoinsCommand = cli.Command{
cli.BoolFlag{
Name: "sweepall",
Usage: "if set, then the amount field will be ignored, " +
"and all the wallet will attempt to sweep all " +
"and the wallet will attempt to sweep all " +
"outputs within the wallet to the target " +
"address",
},
@@ -498,6 +496,14 @@ var connectCommand = cli.Command{
Category: "Peers",
Usage: "Connect to a remote lnd peer.",
ArgsUsage: "<pubkey>@host",
Description: `
Connect to a peer using its <pubkey> and host.
A custom timeout on the connection is supported. For instance, to timeout
the connection request in 30 seconds, use the following:
lncli connect <pubkey>@host --timeout 30s
`,
Flags: []cli.Flag{
cli.BoolFlag{
Name: "perm",
@@ -505,6 +511,13 @@ var connectCommand = cli.Command{
"connect to the target peer.\n" +
" If not, the call will be synchronous.",
},
cli.DurationFlag{
Name: "timeout",
Usage: "The connection timeout value for current request. " +
"Valid uints are {ms, s, m, h}.\n" +
"If not set, the global connection " +
"timeout value (default to 120s) is used.",
},
},
Action: actionDecorator(connectPeer),
}
@@ -526,8 +539,9 @@ func connectPeer(ctx *cli.Context) error {
Host: splitAddr[1],
}
req := &lnrpc.ConnectPeerRequest{
Addr: addr,
Perm: ctx.Bool("perm"),
Addr: addr,
Perm: ctx.Bool("perm"),
Timeout: uint64(ctx.Duration("timeout").Seconds()),
}
lnid, err := client.ConnectPeer(ctxb, req)
@@ -1340,15 +1354,13 @@ mnemonicCheck:
// Additionally, the user may have a passphrase, that will also
// need to be provided so the daemon can properly decipher the
// cipher seed.
fmt.Printf("Input your cipher seed passphrase (press enter if " +
"your seed doesn't have a passphrase): ")
passphrase, err := terminal.ReadPassword(int(syscall.Stdin))
aezeedPass, err = readPassword("Input your cipher seed " +
"passphrase (press enter if your seed doesn't have a " +
"passphrase): ")
if err != nil {
return err
}
aezeedPass = []byte(passphrase)
for {
fmt.Println()
fmt.Printf("Input an optional address look-ahead "+
@@ -1460,12 +1472,10 @@ func capturePassword(instruction string, optional bool,
validate func([]byte) error) ([]byte, error) {
for {
fmt.Printf(instruction)
password, err := terminal.ReadPassword(int(syscall.Stdin))
password, err := readPassword(instruction)
if err != nil {
return nil, err
}
fmt.Println()
// Do not require users to repeat password if
// it is optional and they are not using one.
@@ -1481,21 +1491,16 @@ func capturePassword(instruction string, optional bool,
continue
}
fmt.Printf("Confirm password: ")
passwordConfirmed, err := terminal.ReadPassword(
int(syscall.Stdin),
)
passwordConfirmed, err := readPassword("Confirm password: ")
if err != nil {
return nil, err
}
fmt.Println()
if bytes.Equal(password, passwordConfirmed) {
return password, nil
}
fmt.Println("Passwords don't match, " +
"please try again")
fmt.Println("Passwords don't match, please try again")
fmt.Println()
}
}
@@ -1558,13 +1563,7 @@ func unlock(ctx *cli.Context) error {
// terminal to be a real tty and will fail if a string is piped into
// lncli.
default:
fmt.Printf("Input wallet password: ")
// The variable syscall.Stdin is of a different type in the
// Windows API that's why we need the explicit cast. And of
// course the linter doesn't like it either.
pw, err = terminal.ReadPassword(int(syscall.Stdin)) // nolint:unconvert
fmt.Println()
pw, err = readPassword("Input wallet password: ")
}
if err != nil {
return err
@@ -1625,26 +1624,20 @@ func changePassword(ctx *cli.Context) error {
client, cleanUp := getWalletUnlockerClient(ctx)
defer cleanUp()
fmt.Printf("Input current wallet password: ")
currentPw, err := terminal.ReadPassword(int(syscall.Stdin))
currentPw, err := readPassword("Input current wallet password: ")
if err != nil {
return err
}
fmt.Println()
fmt.Printf("Input new wallet password: ")
newPw, err := terminal.ReadPassword(int(syscall.Stdin))
newPw, err := readPassword("Input new wallet password: ")
if err != nil {
return err
}
fmt.Println()
fmt.Printf("Confirm new wallet password: ")
confirmPw, err := terminal.ReadPassword(int(syscall.Stdin))
confirmPw, err := readPassword("Confirm new wallet password: ")
if err != nil {
return err
}
fmt.Println()
if !bytes.Equal(newPw, confirmPw) {
return fmt.Errorf("passwords don't match")
@@ -2165,6 +2158,11 @@ var queryRoutesCommand = cli.Command{
Name: "use_mc",
Usage: "use mission control probabilities",
},
cli.Uint64Flag{
Name: "outgoing_chanid",
Usage: "(optional) the channel id of the channel " +
"that must be taken to the first hop",
},
cltvLimitFlag,
},
Action: actionDecorator(queryRoutes),
@@ -2217,6 +2215,7 @@ func queryRoutes(ctx *cli.Context) error {
FinalCltvDelta: int32(ctx.Int("final_cltv_delta")),
UseMissionControl: ctx.Bool("use_mc"),
CltvLimit: uint32(ctx.Uint64(cltvLimitFlag.Name)),
OutgoingChanId: ctx.Uint64("outgoing_chanid"),
}
route, err := client.QueryRoutes(ctxb, req)
+162
View File
@@ -0,0 +1,162 @@
package main
import (
"encoding/base64"
"encoding/hex"
"fmt"
"strings"
"github.com/btcsuite/btcwallet/snacl"
"gopkg.in/macaroon.v2"
)
const (
encryptionPrefix = "snacl:"
)
// getPasswordFn is a function that asks the user to type a password after
// presenting it the given prompt.
type getPasswordFn func(prompt string) ([]byte, error)
// macaroonJar is a struct that represents all macaroons of a profile.
type macaroonJar struct {
Default string `json:"default,omitempty"`
Timeout int64 `json:"timeout,omitempty"`
IP string `json:"ip,omitempty"`
Jar []*macaroonEntry `json:"jar"`
}
// macaroonEntry is a struct that represents a single macaroon. Its content can
// either be cleartext (hex encoded) or encrypted (snacl secretbox).
type macaroonEntry struct {
Name string `json:"name"`
Data string `json:"data"`
}
// loadMacaroon returns the fully usable macaroon instance from the entry. This
// detects whether the macaroon needs to be decrypted and does so if necessary.
// An encrypted macaroon that needs to be decrypted will prompt for the user's
// password by calling the provided password callback. Normally that should
// result in the user being prompted for the password in the terminal.
func (e *macaroonEntry) loadMacaroon(
pwCallback getPasswordFn) (*macaroon.Macaroon, error) {
if len(strings.TrimSpace(e.Data)) == 0 {
return nil, fmt.Errorf("macaroon data is empty")
}
var (
macBytes []byte
err error
)
// Either decrypt or simply decode the macaroon data.
if strings.HasPrefix(e.Data, encryptionPrefix) {
parts := strings.Split(e.Data, ":")
if len(parts) != 3 {
return nil, fmt.Errorf("invalid encrypted macaroon " +
"format, expected 'snacl:<key_base64>:" +
"<encrypted_macaroon_base64>'")
}
pw, err := pwCallback("Enter macaroon encryption password: ")
if err != nil {
return nil, fmt.Errorf("could not read password from "+
"terminal: %v", err)
}
macBytes, err = decryptMacaroon(parts[1], parts[2], pw)
if err != nil {
return nil, fmt.Errorf("unable to decrypt macaroon: %v",
err)
}
} else {
macBytes, err = hex.DecodeString(e.Data)
if err != nil {
return nil, fmt.Errorf("unable to hex decode "+
"macaroon: %v", err)
}
}
// Parse the macaroon data into its native struct.
mac := &macaroon.Macaroon{}
if err := mac.UnmarshalBinary(macBytes); err != nil {
return nil, fmt.Errorf("unable to decode macaroon: %v", err)
}
return mac, nil
}
// storeMacaroon stores a native macaroon instance to the entry. If a non-nil
// password is provided, then the macaroon is encrypted with that password. If
// not, the macaroon is stored as plain text.
func (e *macaroonEntry) storeMacaroon(mac *macaroon.Macaroon, pw []byte) error {
// First of all, make sure we can serialize the macaroon.
macBytes, err := mac.MarshalBinary()
if err != nil {
return fmt.Errorf("unable to marshal macaroon: %v", err)
}
if len(pw) == 0 {
e.Data = hex.EncodeToString(macBytes)
return nil
}
// The user did set a password. Let's derive an encryption key from it.
key, err := snacl.NewSecretKey(
&pw, snacl.DefaultN, snacl.DefaultR, snacl.DefaultP,
)
if err != nil {
return fmt.Errorf("unable to create encryption key: %v", err)
}
// Encrypt the macaroon data with the derived key and store it in the
// human readable format snacl:<key_base64>:<encrypted_macaroon_base64>.
encryptedMac, err := key.Encrypt(macBytes)
if err != nil {
return fmt.Errorf("unable to encrypt macaroon: %v", err)
}
keyB64 := base64.StdEncoding.EncodeToString(key.Marshal())
dataB64 := base64.StdEncoding.EncodeToString(encryptedMac)
e.Data = fmt.Sprintf("%s%s:%s", encryptionPrefix, keyB64, dataB64)
return nil
}
// decryptMacaroon decrypts the cipher text macaroon by using the serialized
// encryption key and the password.
func decryptMacaroon(keyB64, dataB64 string, pw []byte) ([]byte, error) {
// Base64 decode both the marshalled encryption key and macaroon data.
keyData, err := base64.StdEncoding.DecodeString(keyB64)
if err != nil {
return nil, fmt.Errorf("could not base64 decode encryption "+
"key: %v", err)
}
encryptedMac, err := base64.StdEncoding.DecodeString(dataB64)
if err != nil {
return nil, fmt.Errorf("could not base64 decode macaroon "+
"data: %v", err)
}
// Unmarshal the encryption key and ask the user for the password.
key := &snacl.SecretKey{}
err = key.Unmarshal(keyData)
if err != nil {
return nil, fmt.Errorf("could not unmarshal encryption key: %v",
err)
}
// Derive the final encryption key and then decrypt the macaroon with
// it.
err = key.DeriveKey(&pw)
if err != nil {
return nil, fmt.Errorf("could not derive encryption key, "+
"possibly due to incorrect password: %v", err)
}
macBytes, err := key.Decrypt(encryptedMac)
if err != nil {
return nil, fmt.Errorf("could not decrypt macaroon data: %v",
err)
}
return macBytes, nil
}
+102
View File
@@ -0,0 +1,102 @@
package main
import (
"encoding/hex"
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/macaroon.v2"
)
var (
dummyMacStr = "0201047465737402067788991234560000062052d26ed139ea5af8" +
"3e675500c4ccb2471f62191b745bab820f129e5588a255d2"
dummyMac, _ = hex.DecodeString(dummyMacStr)
encryptedEntry = &macaroonEntry{
Name: "encryptedMac",
Data: "snacl:exX8xbUOb6Gih88ybL2jZGo+DBDPU2tYKkvo0eVVmbDGDoFP" +
"zlv5xvqNK5eml0LKLcB8LdZRw43qXK1W2OLs/gBAAAAAAAAACAAA" +
"AAAAAAABAAAAAAAAAA==:C8TN/aDOvSLiBCX+IdoPTx+UUWhVdGj" +
"NQvbcaWp+KXQWqPfpRZpjJQ6B2PDx5mJxImcezJGPx8ShAqMdxWe" +
"l2precU+1cOjk7HQFkYuu943eJ00s6JerAY+ssg==",
}
plaintextEntry = &macaroonEntry{
Name: "plaintextMac",
Data: dummyMacStr,
}
testPassword = []byte("S3curePazzw0rd")
pwCallback = func(string) ([]byte, error) {
return testPassword, nil
}
noPwCallback = func(string) ([]byte, error) {
return nil, nil
}
)
// TestMacaroonJarEncrypted tests that a macaroon can be stored and retrieved
// safely by encrypting/decrypting it with a password.
func TestMacaroonJarEncrypted(t *testing.T) {
// Create a new macaroon entry from the dummy macaroon and encrypt it
// with the test password.
newEntry := &macaroonEntry{
Name: "encryptedMac",
}
err := newEntry.storeMacaroon(toMacaroon(t, dummyMac), testPassword)
require.NoError(t, err)
// Now decrypt it again and make sure we get the same content back.
mac, err := newEntry.loadMacaroon(pwCallback)
require.NoError(t, err)
macBytes, err := mac.MarshalBinary()
require.NoError(t, err)
require.Equal(t, dummyMac, macBytes)
// The encrypted data of the entry we just created shouldn't be the
// same as our test entry because of the salt snacl uses.
require.NotEqual(t, encryptedEntry.Data, newEntry.Data)
// Decrypt the hard coded test entry and make sure the decrypted content
// matches our created entry.
mac, err = encryptedEntry.loadMacaroon(pwCallback)
require.NoError(t, err)
macBytes, err = mac.MarshalBinary()
require.NoError(t, err)
require.Equal(t, dummyMac, macBytes)
}
// TestMacaroonJarPlaintext tests that a macaroon can be stored and retrieved
// as plaintext as well.
func TestMacaroonJarPlaintext(t *testing.T) {
// Create a new macaroon entry from the dummy macaroon and encrypt it
// with the test password.
newEntry := &macaroonEntry{
Name: "plaintextMac",
}
err := newEntry.storeMacaroon(toMacaroon(t, dummyMac), nil)
require.NoError(t, err)
// Now decrypt it again and make sure we get the same content back.
mac, err := newEntry.loadMacaroon(noPwCallback)
require.NoError(t, err)
macBytes, err := mac.MarshalBinary()
require.NoError(t, err)
require.Equal(t, dummyMac, macBytes)
require.Equal(t, plaintextEntry.Data, newEntry.Data)
// Load the hard coded plaintext test entry and make sure the loaded
// content matches our created entry.
mac, err = plaintextEntry.loadMacaroon(noPwCallback)
require.NoError(t, err)
macBytes, err = mac.MarshalBinary()
require.NoError(t, err)
require.Equal(t, dummyMac, macBytes)
}
func toMacaroon(t *testing.T, macData []byte) *macaroon.Macaroon {
mac := &macaroon.Macaroon{}
err := mac.UnmarshalBinary(macData)
require.NoError(t, err)
return mac
}
+94 -59
View File
@@ -5,14 +5,12 @@
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"os"
"os/user"
"path/filepath"
"strings"
macaroon "gopkg.in/macaroon.v2"
"syscall"
"github.com/btcsuite/btcutil"
"github.com/lightningnetwork/lnd/build"
@@ -21,6 +19,7 @@ import (
"github.com/lightningnetwork/lnd/macaroons"
"github.com/urfave/cli"
"golang.org/x/crypto/ssh/terminal"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
)
@@ -69,17 +68,30 @@ func getClient(ctx *cli.Context) (lnrpc.LightningClient, func()) {
}
func getClientConn(ctx *cli.Context, skipMacaroons bool) *grpc.ClientConn {
// First, we'll parse the args from the command.
tlsCertPath, macPath, err := extractPathArgs(ctx)
// First, we'll get the selected stored profile or an ephemeral one
// created from the global options in the CLI context.
profile, err := getGlobalOptions(ctx)
if err != nil {
fatal(err)
fatal(fmt.Errorf("could not load global options: %v", err))
}
// Load the specified TLS certificate and build transport credentials
// with it.
creds, err := credentials.NewClientTLSFromFile(tlsCertPath, "")
// Load the specified TLS certificate.
certPool, err := profile.cert()
if err != nil {
fatal(err)
fatal(fmt.Errorf("could not create cert pool: %v", err))
}
// Build transport credentials from the certificate pool. If there is no
// certificate pool, we expect the server to use a non-self-signed
// certificate such as a certificate obtained from Let's Encrypt.
var creds credentials.TransportCredentials
if certPool != nil {
creds = credentials.NewClientTLSFromCert(certPool, "")
} else {
// Fallback to the system pool. Using an empty tls config is an
// alternative to x509.SystemCertPool(). That call is not
// supported on Windows.
creds = credentials.NewTLS(&tls.Config{})
}
// Create a dial options array.
@@ -89,17 +101,31 @@ func getClientConn(ctx *cli.Context, skipMacaroons bool) *grpc.ClientConn {
// Only process macaroon credentials if --no-macaroons isn't set and
// if we're not skipping macaroon processing.
if !ctx.GlobalBool("no-macaroons") && !skipMacaroons {
// Load the specified macaroon file.
macBytes, err := ioutil.ReadFile(macPath)
if err != nil {
fatal(fmt.Errorf("unable to read macaroon path (check "+
"the network setting!): %v", err))
if !profile.NoMacaroons && !skipMacaroons {
// Find out which macaroon to load.
macName := profile.Macaroons.Default
if ctx.GlobalIsSet("macfromjar") {
macName = ctx.GlobalString("macfromjar")
}
var macEntry *macaroonEntry
for _, entry := range profile.Macaroons.Jar {
if entry.Name == macName {
macEntry = entry
break
}
}
if macEntry == nil {
fatal(fmt.Errorf("macaroon with name '%s' not found "+
"in profile", macName))
}
mac := &macaroon.Macaroon{}
if err = mac.UnmarshalBinary(macBytes); err != nil {
fatal(fmt.Errorf("unable to decode macaroon: %v", err))
// Get and possibly decrypt the specified macaroon.
//
// TODO(guggero): Make it possible to cache the password so we
// don't need to ask for it every time.
mac, err := macEntry.loadMacaroon(readPassword)
if err != nil {
fatal(fmt.Errorf("could not load macaroon: %v", err))
}
macConstraints := []macaroons.Constraint{
@@ -114,16 +140,18 @@ func getClientConn(ctx *cli.Context, skipMacaroons bool) *grpc.ClientConn {
// altogether if, in the latter case, this time is more than 60
// seconds).
// TODO(aakselrod): add better anti-replay protection.
macaroons.TimeoutConstraint(ctx.GlobalInt64("macaroontimeout")),
macaroons.TimeoutConstraint(profile.Macaroons.Timeout),
// Lock macaroon down to a specific IP address.
macaroons.IPLockConstraint(ctx.GlobalString("macaroonip")),
macaroons.IPLockConstraint(profile.Macaroons.IP),
// ... Add more constraints if needed.
}
// Apply constraints to the macaroon.
constrainedMac, err := macaroons.AddConstraints(mac, macConstraints...)
constrainedMac, err := macaroons.AddConstraints(
mac, macConstraints...,
)
if err != nil {
fatal(err)
}
@@ -139,7 +167,7 @@ func getClientConn(ctx *cli.Context, skipMacaroons bool) *grpc.ClientConn {
opts = append(opts, grpc.WithContextDialer(genericDialer))
opts = append(opts, grpc.WithDefaultCallOptions(maxMsgRecvSize))
conn, err := grpc.Dial(ctx.GlobalString("rpcserver"), opts...)
conn, err := grpc.Dial(profile.RPCServer, opts...)
if err != nil {
fatal(fmt.Errorf("unable to connect to RPC server: %v", err))
}
@@ -171,13 +199,13 @@ func extractPathArgs(ctx *cli.Context) (string, string, error) {
// properly read the macaroons (if needed) and also the cert. This will
// either be the default, or will have been overwritten by the end
// user.
lndDir := cleanAndExpandPath(ctx.GlobalString("lnddir"))
lndDir := lncfg.CleanAndExpandPath(ctx.GlobalString("lnddir"))
// If the macaroon path as been manually provided, then we'll only
// target the specified file.
var macPath string
if ctx.GlobalString("macaroonpath") != "" {
macPath = cleanAndExpandPath(ctx.GlobalString("macaroonpath"))
macPath = lncfg.CleanAndExpandPath(ctx.GlobalString("macaroonpath"))
} else {
// Otherwise, we'll go into the path:
// lnddir/data/chain/<chain>/<network> in order to fetch the
@@ -188,7 +216,7 @@ func extractPathArgs(ctx *cli.Context) (string, string, error) {
)
}
tlsCertPath := cleanAndExpandPath(ctx.GlobalString("tlscertpath"))
tlsCertPath := lncfg.CleanAndExpandPath(ctx.GlobalString("tlscertpath"))
// If a custom lnd directory was set, we'll also check if custom paths
// for the TLS cert and macaroon file were set as well. If not, we'll
@@ -211,45 +239,60 @@ func main() {
cli.StringFlag{
Name: "rpcserver",
Value: defaultRPCHostPort,
Usage: "host:port of ln daemon",
Usage: "The host:port of LN daemon.",
},
cli.StringFlag{
Name: "lnddir",
Value: defaultLndDir,
Usage: "path to lnd's base directory",
Usage: "The path to lnd's base directory.",
},
cli.StringFlag{
Name: "tlscertpath",
Value: defaultTLSCertPath,
Usage: "path to TLS certificate",
Usage: "The path to lnd's TLS certificate.",
},
cli.StringFlag{
Name: "chain, c",
Usage: "the chain lnd is running on e.g. bitcoin",
Usage: "The chain lnd is running on, e.g. bitcoin.",
Value: "bitcoin",
},
cli.StringFlag{
Name: "network, n",
Usage: "the network lnd is running on e.g. mainnet, " +
Usage: "The network lnd is running on, e.g. mainnet, " +
"testnet, etc.",
Value: "mainnet",
},
cli.BoolFlag{
Name: "no-macaroons",
Usage: "disable macaroon authentication",
Usage: "Disable macaroon authentication.",
},
cli.StringFlag{
Name: "macaroonpath",
Usage: "path to macaroon file",
Usage: "The path to macaroon file.",
},
cli.Int64Flag{
Name: "macaroontimeout",
Value: 60,
Usage: "anti-replay macaroon validity time in seconds",
Usage: "Anti-replay macaroon validity time in seconds.",
},
cli.StringFlag{
Name: "macaroonip",
Usage: "if set, lock macaroon to specific IP address",
Usage: "If set, lock macaroon to specific IP address.",
},
cli.StringFlag{
Name: "profile, p",
Usage: "Instead of reading settings from command " +
"line parameters or using the default " +
"profile, use a specific profile. If " +
"a default profile is set, this flag can be " +
"set to an empty string to disable reading " +
"values from the profiles file.",
},
cli.StringFlag{
Name: "macfromjar",
Usage: "Use this macaroon from the profile's " +
"macaroon jar instead of the default one. " +
"Can only be used if profiles are defined.",
},
}
app.Commands = []cli.Command{
@@ -301,8 +344,13 @@ func main() {
verifyChanBackupCommand,
restoreChanBackupCommand,
bakeMacaroonCommand,
listMacaroonIDsCommand,
deleteMacaroonIDCommand,
listPermissionsCommand,
printMacaroonCommand,
trackPaymentCommand,
versionCommand,
profileSubCommand,
}
// Add any extra commands determined by build flags.
@@ -318,28 +366,15 @@ func main() {
}
}
// cleanAndExpandPath expands environment variables and leading ~ in the
// passed path, cleans the result, and returns it.
// This function is taken from https://github.com/btcsuite/btcd
func cleanAndExpandPath(path string) string {
if path == "" {
return ""
}
// readPassword reads a password from the terminal. This requires there to be an
// actual TTY so passing in a password from stdin won't work.
func readPassword(text string) ([]byte, error) {
fmt.Print(text)
// Expand initial ~ to OS specific home directory.
if strings.HasPrefix(path, "~") {
var homeDir string
user, err := user.Current()
if err == nil {
homeDir = user.HomeDir
} else {
homeDir = os.Getenv("HOME")
}
path = strings.Replace(path, "~", homeDir, 1)
}
// NOTE: The os.ExpandEnv doesn't work with Windows-style %VARIABLE%,
// but the variables can still be expanded via POSIX-style $VARIABLE.
return filepath.Clean(os.ExpandEnv(path))
// The variable syscall.Stdin is of a different type in the Windows API
// that's why we need the explicit cast. And of course the linter
// doesn't like it either.
pw, err := terminal.ReadPassword(int(syscall.Stdin)) // nolint:unconvert
fmt.Println()
return pw, err
}
+245
View File
@@ -0,0 +1,245 @@
package main
import (
"bytes"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"path"
"strings"
"github.com/lightningnetwork/lnd/lncfg"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/walletunlocker"
"github.com/urfave/cli"
"gopkg.in/macaroon.v2"
)
var (
errNoProfileFile = errors.New("no profile file found")
)
// profileEntry is a struct that represents all settings for one specific
// profile.
type profileEntry struct {
Name string `json:"name"`
RPCServer string `json:"rpcserver"`
LndDir string `json:"lnddir"`
Chain string `json:"chain"`
Network string `json:"network"`
NoMacaroons bool `json:"no-macaroons,omitempty"`
TLSCert string `json:"tlscert"`
Macaroons *macaroonJar `json:"macaroons"`
}
// cert returns the profile's TLS certificate as a x509 certificate pool.
func (e *profileEntry) cert() (*x509.CertPool, error) {
if e.TLSCert == "" {
return nil, nil
}
cp := x509.NewCertPool()
if !cp.AppendCertsFromPEM([]byte(e.TLSCert)) {
return nil, fmt.Errorf("credentials: failed to append " +
"certificate")
}
return cp, nil
}
// getGlobalOptions returns the global connection options. If a profile file
// exists, these global options might be read from a predefined profile. If no
// profile exists, the global options from the command line are returned as an
// ephemeral profile entry.
func getGlobalOptions(ctx *cli.Context) (*profileEntry, error) {
var profileName string
// Try to load the default profile file and depending on its existence
// what profile to use.
f, err := loadProfileFile(defaultProfileFile)
switch {
// The legacy case where no profile file exists and the user also didn't
// request to use one. We only consider the global options here.
case err == errNoProfileFile && !ctx.GlobalIsSet("profile"):
return profileFromContext(ctx, false)
// The file doesn't exist but the user specified an explicit profile.
case err == errNoProfileFile && ctx.GlobalIsSet("profile"):
return nil, fmt.Errorf("profile file %s does not exist",
defaultProfileFile)
// There is a file but we couldn't read/parse it.
case err != nil:
return nil, fmt.Errorf("could not read profile file %s: "+
"%v", defaultProfileFile, err)
// The user explicitly disabled the use of profiles for this command by
// setting the flag to an empty string. We fall back to the default/old
// behavior.
case ctx.GlobalIsSet("profile") && ctx.GlobalString("profile") == "":
return profileFromContext(ctx, false)
// There is a file, but no default profile is specified. The user also
// didn't specify a profile to use so we fall back to the default/old
// behavior.
case !ctx.GlobalIsSet("profile") && len(f.Default) == 0:
return profileFromContext(ctx, false)
// The user didn't specify a profile but there is a default one defined.
case !ctx.GlobalIsSet("profile") && len(f.Default) > 0:
profileName = f.Default
// The user specified a specific profile to use.
case ctx.GlobalIsSet("profile"):
profileName = ctx.GlobalString("profile")
}
// If we got to here, we do have a profile file and know the name of the
// profile to use. Now we just need to make sure it does exist.
for _, prof := range f.Profiles {
if prof.Name == profileName {
return prof, nil
}
}
return nil, fmt.Errorf("profile '%s' not found in file %s", profileName,
defaultProfileFile)
}
// profileFromContext creates an ephemeral profile entry from the global options
// set in the CLI context.
func profileFromContext(ctx *cli.Context, store bool) (*profileEntry, error) {
// Parse the paths of the cert and macaroon. This will validate the
// chain and network value as well.
tlsCertPath, macPath, err := extractPathArgs(ctx)
if err != nil {
return nil, err
}
// Load the certificate file now, if specified. We store it as plain PEM
// directly.
var tlsCert []byte
if lnrpc.FileExists(tlsCertPath) {
var err error
tlsCert, err = ioutil.ReadFile(tlsCertPath)
if err != nil {
return nil, fmt.Errorf("could not load TLS cert file "+
"%s: %v", tlsCertPath, err)
}
}
// Now load and possibly encrypt the macaroon file.
macBytes, err := ioutil.ReadFile(macPath)
if err != nil {
return nil, fmt.Errorf("unable to read macaroon path (check "+
"the network setting!): %v", err)
}
mac := &macaroon.Macaroon{}
if err = mac.UnmarshalBinary(macBytes); err != nil {
return nil, fmt.Errorf("unable to decode macaroon: %v", err)
}
var pw []byte
if store {
// Read a password from the terminal. If it's empty, we won't
// encrypt the macaroon and store it plaintext.
pw, err = capturePassword(
"Enter password to encrypt macaroon with or leave "+
"blank to store in plaintext: ", true,
walletunlocker.ValidatePassword,
)
if err != nil {
return nil, fmt.Errorf("unable to get encryption "+
"password: %v", err)
}
}
macEntry := &macaroonEntry{}
if err = macEntry.storeMacaroon(mac, pw); err != nil {
return nil, fmt.Errorf("unable to store macaroon: %v", err)
}
// We determine the name of the macaroon from the file itself but cut
// off the ".macaroon" at the end.
macEntry.Name = path.Base(macPath)
if path.Ext(macEntry.Name) == "macaroon" {
macEntry.Name = strings.TrimSuffix(macEntry.Name, ".macaroon")
}
// Now that we have the complicated arguments behind us, let's return
// the new entry with all the values populated.
return &profileEntry{
RPCServer: ctx.GlobalString("rpcserver"),
LndDir: lncfg.CleanAndExpandPath(ctx.GlobalString("lnddir")),
Chain: ctx.GlobalString("chain"),
Network: ctx.GlobalString("network"),
NoMacaroons: ctx.GlobalBool("no-macaroons"),
TLSCert: string(tlsCert),
Macaroons: &macaroonJar{
Default: macEntry.Name,
Timeout: ctx.GlobalInt64("macaroontimeout"),
IP: ctx.GlobalString("macaroonip"),
Jar: []*macaroonEntry{macEntry},
},
}, nil
}
// loadProfileFile tries to load the file specified and JSON deserialize it into
// the profile file struct.
func loadProfileFile(file string) (*profileFile, error) {
if !lnrpc.FileExists(file) {
return nil, errNoProfileFile
}
content, err := ioutil.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("could not load profile file %s: %v",
file, err)
}
f := &profileFile{}
err = f.unmarshalJSON(content)
if err != nil {
return nil, fmt.Errorf("could not unmarshal profile file %s: "+
"%v", file, err)
}
return f, nil
}
// saveProfileFile stores the given profile file struct in the specified file,
// overwriting it if it already existed.
func saveProfileFile(file string, f *profileFile) error {
content, err := f.marshalJSON()
if err != nil {
return fmt.Errorf("could not marshal profile: %v", err)
}
return ioutil.WriteFile(file, content, 0644)
}
// profileFile is a struct that represents the whole content of a profile file.
type profileFile struct {
Default string `json:"default,omitempty"`
Profiles []*profileEntry `json:"profiles"`
}
// unmarshalJSON tries to parse the given JSON and unmarshal it into the
// receiving instance.
func (f *profileFile) unmarshalJSON(content []byte) error {
return json.Unmarshal(content, f)
}
// marshalJSON serializes the receiving instance to formatted/indented JSON.
func (f *profileFile) marshalJSON() ([]byte, error) {
b, err := json.Marshal(f)
if err != nil {
return nil, fmt.Errorf("error JSON marshalling profile: %v",
err)
}
var out bytes.Buffer
err = json.Indent(&out, b, "", " ")
if err != nil {
return nil, fmt.Errorf("error indenting profile JSON: %v", err)
}
out.WriteString("\n")
return out.Bytes(), nil
}
+15 -10
View File
@@ -14,23 +14,28 @@ func main() {
// function will also set up logging properly.
loadedConfig, err := lnd.LoadConfig()
if err != nil {
if e, ok := err.(*flags.Error); !ok || e.Type != flags.ErrHelp {
// Print error if not due to help request.
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
// Help was requested, exit normally.
os.Exit(0)
}
// Hook interceptor for os signals.
if err := signal.Intercept(); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
// Hook interceptor for os signals.
signal.Intercept()
// Call the "real" main in a nested manner so the defers will properly
// be executed in the case of a graceful shutdown.
err = lnd.Main(
if err := lnd.Main(
loadedConfig, lnd.ListenerCfg{}, signal.ShutdownChannel(),
)
if err != nil {
if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
} else {
_, _ = fmt.Fprintln(os.Stderr, err)
}
); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
+214 -65
View File
@@ -27,6 +27,7 @@ import (
"github.com/lightningnetwork/lnd/discovery"
"github.com/lightningnetwork/lnd/htlcswitch"
"github.com/lightningnetwork/lnd/htlcswitch/hodl"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lncfg"
"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
"github.com/lightningnetwork/lnd/lnrpc/signrpc"
@@ -63,6 +64,8 @@ const (
defaultMaxLogFileSize = 10
defaultMinBackoff = time.Second
defaultMaxBackoff = time.Hour
defaultLetsEncryptDirname = "letsencrypt"
defaultLetsEncryptListen = ":80"
defaultTorSOCKSPort = 9050
defaultTorDNSHost = "soa.nodes.lightning.directory"
@@ -86,6 +89,27 @@ const (
// HostAnnouncer will wait between DNS resolutions to check if the
// backing IP of a host has changed.
defaultHostSampleInterval = time.Minute * 5
defaultChainInterval = time.Minute
defaultChainTimeout = time.Second * 10
defaultChainBackoff = time.Second * 30
defaultChainAttempts = 3
// Set defaults for a health check which ensures that we have space
// available on disk. Although this check is off by default so that we
// avoid breaking any existing setups (particularly on mobile), we still
// set the other default values so that the health check can be easily
// enabled with sane defaults.
defaultRequiredDisk = 0.1
defaultDiskInterval = time.Hour * 12
defaultDiskTimeout = time.Second * 5
defaultDiskBackoff = time.Minute
defaultDiskAttempts = 0
// defaultRemoteMaxHtlcs specifies the default limit for maximum
// concurrent HTLCs the remote party may add to commitment transactions.
// This value can be overridden with --default-remote-max-htlcs.
defaultRemoteMaxHtlcs = 483
)
var (
@@ -106,8 +130,9 @@ var (
defaultTowerDir = filepath.Join(defaultDataDir, defaultTowerSubDirname)
defaultTLSCertPath = filepath.Join(DefaultLndDir, defaultTLSCertFilename)
defaultTLSKeyPath = filepath.Join(DefaultLndDir, defaultTLSKeyFilename)
defaultTLSCertPath = filepath.Join(DefaultLndDir, defaultTLSCertFilename)
defaultTLSKeyPath = filepath.Join(DefaultLndDir, defaultTLSKeyFilename)
defaultLetsEncryptDir = filepath.Join(DefaultLndDir, defaultLetsEncryptDirname)
defaultBtcdDir = btcutil.AppDataDir("btcd", false)
defaultBtcdRPCCertFile = filepath.Join(defaultBtcdDir, "rpc.cert")
@@ -126,6 +151,8 @@ var (
// estimatesmartfee RPC call.
defaultBitcoindEstimateMode = "CONSERVATIVE"
bitcoindEstimateModes = [2]string{"ECONOMICAL", defaultBitcoindEstimateMode}
defaultSphinxDbName = "sphinxreplay.db"
)
// Config defines the configuration options for lnd.
@@ -140,13 +167,14 @@ type Config struct {
DataDir string `short:"b" long:"datadir" description:"The directory to store lnd's data within"`
SyncFreelist bool `long:"sync-freelist" description:"Whether the databases used within lnd should sync their freelist to disk. This is disabled by default resulting in improved memory performance during operation, but with an increase in startup time."`
TLSCertPath string `long:"tlscertpath" description:"Path to write the TLS certificate for lnd's RPC and REST services"`
TLSKeyPath string `long:"tlskeypath" description:"Path to write the TLS private key for lnd's RPC and REST services"`
TLSExtraIPs []string `long:"tlsextraip" description:"Adds an extra ip to the generated certificate"`
TLSExtraDomains []string `long:"tlsextradomain" description:"Adds an extra domain to the generated certificate"`
TLSAutoRefresh bool `long:"tlsautorefresh" description:"Re-generate TLS certificate and key if the IPs or domains are changed"`
TLSCertPath string `long:"tlscertpath" description:"Path to write the TLS certificate for lnd's RPC and REST services"`
TLSKeyPath string `long:"tlskeypath" description:"Path to write the TLS private key for lnd's RPC and REST services"`
TLSExtraIPs []string `long:"tlsextraip" description:"Adds an extra ip to the generated certificate"`
TLSExtraDomains []string `long:"tlsextradomain" description:"Adds an extra domain to the generated certificate"`
TLSAutoRefresh bool `long:"tlsautorefresh" description:"Re-generate TLS certificate and key if the IPs or domains are changed"`
TLSDisableAutofill bool `long:"tlsdisableautofill" description:"Do not include the interface IPs or the system hostname in TLS certificate, use first --tlsextradomain as Common Name instead, if set"`
NoMacaroons bool `long:"no-macaroons" description:"Disable macaroon authentication"`
NoMacaroons bool `long:"no-macaroons" description:"Disable macaroon authentication, can only be used if server is not listening on a public interface."`
AdminMacPath string `long:"adminmacaroonpath" description:"Path to write the admin macaroon for lnd's RPC and REST services if it doesn't exist"`
ReadMacPath string `long:"readonlymacaroonpath" description:"Path to write the read-only macaroon for lnd's RPC and REST services if it doesn't exist"`
InvoiceMacPath string `long:"invoicemacaroonpath" description:"Path to the invoice-only macaroon for lnd's RPC and REST services if it doesn't exist"`
@@ -155,25 +183,30 @@ type Config struct {
MaxLogFileSize int `long:"maxlogfilesize" description:"Maximum logfile size in MB"`
AcceptorTimeout time.Duration `long:"acceptortimeout" description:"Time after which an RPCAcceptor will time out and return false if it hasn't yet received a response"`
LetsEncryptDir string `long:"letsencryptdir" description:"The directory to store Let's Encrypt certificates within"`
LetsEncryptListen string `long:"letsencryptlisten" description:"The IP:port on which lnd will listen for Let's Encrypt challenges. Let's Encrypt will always try to contact on port 80. Often non-root processes are not allowed to bind to ports lower than 1024. This configuration option allows a different port to be used, but must be used in combination with port forwarding from port 80. This configuration can also be used to specify another IP address to listen on, for example an IPv6 address."`
LetsEncryptDomain string `long:"letsencryptdomain" description:"Request a Let's Encrypt certificate for this domain. Note that the certicate is only requested and stored when the first rpc connection comes in."`
// We'll parse these 'raw' string arguments into real net.Addrs in the
// loadConfig function. We need to expose the 'raw' strings so the
// command line library can access them.
// Only the parsed net.Addrs should be used!
RawRPCListeners []string `long:"rpclisten" description:"Add an interface/port/socket to listen for RPC connections"`
RawRESTListeners []string `long:"restlisten" description:"Add an interface/port/socket to listen for REST connections"`
RawListeners []string `long:"listen" description:"Add an interface/port to listen for peer connections"`
RawExternalIPs []string `long:"externalip" description:"Add an ip:port to the list of local addresses we claim to listen on to peers. If a port is not specified, the default (9735) will be used regardless of other parameters"`
ExternalHosts []string `long:"externalhosts" description:"A set of hosts that should be periodically resolved to announce IPs for"`
RPCListeners []net.Addr
RESTListeners []net.Addr
RestCORS []string `long:"restcors" description:"Add an ip:port/hostname to allow cross origin access from. To allow all origins, set as \"*\"."`
Listeners []net.Addr
ExternalIPs []net.Addr
DisableListen bool `long:"nolisten" description:"Disable listening for incoming peer connections"`
DisableRest bool `long:"norest" description:"Disable REST API"`
NAT bool `long:"nat" description:"Toggle NAT traversal support (using either UPnP or NAT-PMP) to automatically advertise your external IP address to the network -- NOTE this does not support devices behind multiple NATs"`
MinBackoff time.Duration `long:"minbackoff" description:"Shortest backoff when reconnecting to persistent peers. Valid time units are {s, m, h}."`
MaxBackoff time.Duration `long:"maxbackoff" description:"Longest backoff when reconnecting to persistent peers. Valid time units are {s, m, h}."`
RawRPCListeners []string `long:"rpclisten" description:"Add an interface/port/socket to listen for RPC connections"`
RawRESTListeners []string `long:"restlisten" description:"Add an interface/port/socket to listen for REST connections"`
RawListeners []string `long:"listen" description:"Add an interface/port to listen for peer connections"`
RawExternalIPs []string `long:"externalip" description:"Add an ip:port to the list of local addresses we claim to listen on to peers. If a port is not specified, the default (9735) will be used regardless of other parameters"`
ExternalHosts []string `long:"externalhosts" description:"A set of hosts that should be periodically resolved to announce IPs for"`
RPCListeners []net.Addr
RESTListeners []net.Addr
RestCORS []string `long:"restcors" description:"Add an ip:port/hostname to allow cross origin access from. To allow all origins, set as \"*\"."`
Listeners []net.Addr
ExternalIPs []net.Addr
DisableListen bool `long:"nolisten" description:"Disable listening for incoming peer connections"`
DisableRest bool `long:"norest" description:"Disable REST API"`
NAT bool `long:"nat" description:"Toggle NAT traversal support (using either UPnP or NAT-PMP) to automatically advertise your external IP address to the network -- NOTE this does not support devices behind multiple NATs"`
MinBackoff time.Duration `long:"minbackoff" description:"Shortest backoff when reconnecting to persistent peers. Valid time units are {s, m, h}."`
MaxBackoff time.Duration `long:"maxbackoff" description:"Longest backoff when reconnecting to persistent peers. Valid time units are {s, m, h}."`
ConnectionTimeout time.Duration `long:"connectiontimeout" description:"The timeout value for network connections. Valid time units are {ms, s, m, h}."`
DebugLevel string `short:"d" long:"debuglevel" description:"Logging level for all subsystems {trace, debug, info, warn, error, critical} -- You may also specify <subsystem>=<level>,<subsystem2>=<level>,... to set the log level for individual subsystems -- Use show to list available subsystems"`
@@ -186,6 +219,8 @@ type Config struct {
MaxPendingChannels int `long:"maxpendingchannels" description:"The maximum number of incoming pending channels permitted per peer."`
BackupFilePath string `long:"backupfilepath" description:"The target location of the channel backup file"`
FeeURL string `long:"feeurl" description:"Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network."`
Bitcoin *lncfg.Chain `group:"Bitcoin" namespace:"bitcoin"`
BtcdMode *lncfg.Btcd `group:"btcd" namespace:"btcd"`
BitcoindMode *lncfg.Bitcoind `group:"bitcoind" namespace:"bitcoind"`
@@ -205,7 +240,7 @@ type Config struct {
NoNetBootstrap bool `long:"nobootstrap" description:"If true, then automatic network bootstrapping will not be attempted."`
NoSeedBackup bool `long:"noseedbackup" description:"If true, NO SEED WILL BE EXPOSED AND THE WALLET WILL BE ENCRYPTED USING THE DEFAULT PASSPHRASE -- EVER. THIS FLAG IS ONLY FOR TESTING AND IS BEING DEPRECATED."`
NoSeedBackup bool `long:"noseedbackup" description:"If true, NO SEED WILL BE EXPOSED -- EVER, AND THE WALLET WILL BE ENCRYPTED USING THE DEFAULT PASSPHRASE. THIS FLAG IS ONLY FOR TESTING AND SHOULD NEVER BE USED ON MAINNET."`
PaymentsExpirationGracePeriod time.Duration `long:"payments-expiration-grace-period" description:"A period to wait before force closing channels with outgoing htlcs that have timed-out and are a result of this node initiated payments."`
TrickleDelay int `long:"trickledelay" description:"Time in milliseconds between each release of announcements to the network"`
@@ -216,6 +251,9 @@ type Config struct {
Alias string `long:"alias" description:"The node alias. Used as a moniker by peers and intelligence services"`
Color string `long:"color" description:"The color of the node in hex format (i.e. '#3399FF'). Used to customize node appearance in intelligence services"`
MinChanSize int64 `long:"minchansize" description:"The smallest channel size (in satoshis) that we should accept. Incoming channels smaller than this will be rejected"`
MaxChanSize int64 `long:"maxchansize" description:"The largest channel size (in satoshis) that we should accept. Incoming channels larger than this will be rejected"`
DefaultRemoteMaxHtlcs uint16 `long:"default-remote-max-htlcs" description:"The default max_htlc applied when opening or accepting channels. This value limits the number of concurrent HTLCs that the remote party can add to the commitment. The maximum possible value is 483."`
NumGraphSyncPeers int `long:"numgraphsyncpeers" description:"The number of peers that we should receive new graph updates from. This option can be tuned to save bandwidth for light clients or routing nodes."`
HistoricalSyncInterval time.Duration `long:"historicalsyncinterval" description:"The polling interval between historical graph sync attempts. Each historical graph sync attempt ensures we reconcile with the remote peer's graph from the genesis block."`
@@ -242,6 +280,10 @@ type Config struct {
KeysendHoldTime time.Duration `long:"keysend-hold-time" description:"If non-zero, keysend payments are accepted but not immediately settled. If the payment isn't settled manually after the specified time, it is canceled automatically. [experimental]"`
GcCanceledInvoicesOnStartup bool `long:"gc-canceled-invoices-on-startup" description:"If true, we'll attempt to garbage collect canceled invoices upon start."`
GcCanceledInvoicesOnTheFly bool `long:"gc-canceled-invoices-on-the-fly" description:"If true, we'll delete newly canceled invoices on the fly."`
Routing *routing.Conf `group:"routing" namespace:"routing"`
Workers *lncfg.Workers `group:"workers" namespace:"workers"`
@@ -258,6 +300,8 @@ type Config struct {
AllowCircularRoute bool `long:"allow-circular-route" description:"If true, our node will allow htlc forwards that arrive and depart on the same channel."`
HealthChecks *lncfg.HealthCheckConfig `group:"healthcheck" namespace:"healthcheck"`
DB *lncfg.DB `group:"db" namespace:"db"`
// LogWriter is the root logger that all of the daemon's subloggers are
@@ -272,21 +316,26 @@ type Config struct {
// network. This path will hold the files related to each different
// network.
networkDir string
// ActiveNetParams contains parameters of the target chain.
ActiveNetParams bitcoinNetParams
}
// DefaultConfig returns all default values for the Config struct.
func DefaultConfig() Config {
return Config{
LndDir: DefaultLndDir,
ConfigFile: DefaultConfigFile,
DataDir: defaultDataDir,
DebugLevel: defaultLogLevel,
TLSCertPath: defaultTLSCertPath,
TLSKeyPath: defaultTLSKeyPath,
LogDir: defaultLogDir,
MaxLogFiles: defaultMaxLogFiles,
MaxLogFileSize: defaultMaxLogFileSize,
AcceptorTimeout: defaultAcceptorTimeout,
LndDir: DefaultLndDir,
ConfigFile: DefaultConfigFile,
DataDir: defaultDataDir,
DebugLevel: defaultLogLevel,
TLSCertPath: defaultTLSCertPath,
TLSKeyPath: defaultTLSKeyPath,
LetsEncryptDir: defaultLetsEncryptDir,
LetsEncryptListen: defaultLetsEncryptListen,
LogDir: defaultLogDir,
MaxLogFiles: defaultMaxLogFiles,
MaxLogFileSize: defaultMaxLogFileSize,
AcceptorTimeout: defaultAcceptorTimeout,
Bitcoin: &lncfg.Chain{
MinHTLCIn: defaultBitcoinMinHTLCInMSat,
MinHTLCOut: defaultBitcoinMinHTLCOutMSat,
@@ -328,6 +377,7 @@ func DefaultConfig() Config {
NoSeedBackup: defaultNoSeedBackup,
MinBackoff: defaultMinBackoff,
MaxBackoff: defaultMaxBackoff,
ConnectionTimeout: tor.DefaultConnTimeout,
SubRPCServers: &subRPCServerConfigs{
SignRPC: &signrpc.Config{},
RouterRPC: routerrpc.DefaultConfig(),
@@ -352,6 +402,8 @@ func DefaultConfig() Config {
Alias: defaultAlias,
Color: defaultColor,
MinChanSize: int64(minChanFundingSize),
MaxChanSize: int64(0),
DefaultRemoteMaxHtlcs: defaultRemoteMaxHtlcs,
NumGraphSyncPeers: defaultMinPeers,
HistoricalSyncInterval: discovery.DefaultHistoricalSyncInterval,
Tor: &lncfg.Tor{
@@ -373,11 +425,29 @@ func DefaultConfig() Config {
Watchtower: &lncfg.Watchtower{
TowerDir: defaultTowerDir,
},
HealthChecks: &lncfg.HealthCheckConfig{
ChainCheck: &lncfg.CheckConfig{
Interval: defaultChainInterval,
Timeout: defaultChainTimeout,
Attempts: defaultChainAttempts,
Backoff: defaultChainBackoff,
},
DiskCheck: &lncfg.DiskCheckConfig{
RequiredRemaining: defaultRequiredDisk,
CheckConfig: &lncfg.CheckConfig{
Interval: defaultDiskInterval,
Attempts: defaultDiskAttempts,
Timeout: defaultDiskTimeout,
Backoff: defaultDiskBackoff,
},
},
},
MaxOutgoingCltvExpiry: htlcswitch.DefaultMaxOutgoingCltvExpiry,
MaxChannelFeeAllocation: htlcswitch.DefaultMaxLinkFeeAllocation,
LogWriter: build.NewRotatingLogWriter(),
DB: lncfg.DefaultDB(),
registeredChains: newChainRegistry(),
ActiveNetParams: bitcoinTestNetParams,
}
}
@@ -466,6 +536,9 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
lndDir := CleanAndExpandPath(cfg.LndDir)
if lndDir != DefaultLndDir {
cfg.DataDir = filepath.Join(lndDir, defaultDataDirname)
cfg.LetsEncryptDir = filepath.Join(
lndDir, defaultLetsEncryptDirname,
)
cfg.TLSCertPath = filepath.Join(lndDir, defaultTLSCertFilename)
cfg.TLSKeyPath = filepath.Join(lndDir, defaultTLSKeyFilename)
cfg.LogDir = filepath.Join(lndDir, defaultLogDirname)
@@ -479,23 +552,28 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
}
}
// Create the lnd directory if it doesn't already exist.
funcName := "loadConfig"
if err := os.MkdirAll(lndDir, 0700); err != nil {
// Show a nicer error message if it's because a symlink is
// linked to a directory that does not exist (probably because
// it's not mounted).
if e, ok := err.(*os.PathError); ok && os.IsExist(err) {
if link, lerr := os.Readlink(e.Path); lerr == nil {
str := "is symlink %s -> %s mounted?"
err = fmt.Errorf(str, e.Path, link)
makeDirectory := func(dir string) error {
err := os.MkdirAll(dir, 0700)
if err != nil {
// Show a nicer error message if it's because a symlink
// is linked to a directory that does not exist
// (probably because it's not mounted).
if e, ok := err.(*os.PathError); ok && os.IsExist(err) {
link, lerr := os.Readlink(e.Path)
if lerr == nil {
str := "is symlink %s -> %s mounted?"
err = fmt.Errorf(str, e.Path, link)
}
}
str := "%s: Failed to create lnd directory: %v"
err := fmt.Errorf(str, funcName, err)
_, _ = fmt.Fprintln(os.Stderr, err)
return err
}
str := "%s: Failed to create lnd directory: %v"
err := fmt.Errorf(str, funcName, err)
_, _ = fmt.Fprintln(os.Stderr, err)
return nil, err
return nil
}
// As soon as we're done parsing configuration options, ensure all paths
@@ -504,6 +582,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
cfg.DataDir = CleanAndExpandPath(cfg.DataDir)
cfg.TLSCertPath = CleanAndExpandPath(cfg.TLSCertPath)
cfg.TLSKeyPath = CleanAndExpandPath(cfg.TLSKeyPath)
cfg.LetsEncryptDir = CleanAndExpandPath(cfg.LetsEncryptDir)
cfg.AdminMacPath = CleanAndExpandPath(cfg.AdminMacPath)
cfg.ReadMacPath = CleanAndExpandPath(cfg.ReadMacPath)
cfg.InvoiceMacPath = CleanAndExpandPath(cfg.InvoiceMacPath)
@@ -516,6 +595,24 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
cfg.Tor.WatchtowerKeyPath = CleanAndExpandPath(cfg.Tor.WatchtowerKeyPath)
cfg.Watchtower.TowerDir = CleanAndExpandPath(cfg.Watchtower.TowerDir)
// Create the lnd directory and all other sub directories if they don't
// already exist. This makes sure that directory trees are also created
// for files that point to outside of the lnddir.
dirs := []string{
lndDir, cfg.DataDir,
cfg.LetsEncryptDir, cfg.Watchtower.TowerDir,
filepath.Dir(cfg.TLSCertPath), filepath.Dir(cfg.TLSKeyPath),
filepath.Dir(cfg.AdminMacPath), filepath.Dir(cfg.ReadMacPath),
filepath.Dir(cfg.InvoiceMacPath),
filepath.Dir(cfg.Tor.PrivateKeyPath),
filepath.Dir(cfg.Tor.WatchtowerKeyPath),
}
for _, dir := range dirs {
if err := makeDirectory(dir); err != nil {
return nil, err
}
}
// Ensure that the user didn't attempt to specify negative values for
// any of the autopilot params.
if cfg.Autopilot.MaxChannels < 0 {
@@ -556,7 +653,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
}
// Ensure that the specified values for the min and max channel size
// don't are within the bounds of the normal chan size constraints.
// are within the bounds of the normal chan size constraints.
if cfg.Autopilot.MinChannelSize < int64(minChanFundingSize) {
cfg.Autopilot.MinChannelSize = int64(minChanFundingSize)
}
@@ -568,6 +665,38 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
return nil, err
}
// Ensure that --maxchansize is properly handled when set by user.
// For non-Wumbo channels this limit remains 16777215 satoshis by default
// as specified in BOLT-02. For wumbo channels this limit is 1,000,000,000.
// satoshis (10 BTC). Always enforce --maxchansize explicitly set by user.
// If unset (marked by 0 value), then enforce proper default.
if cfg.MaxChanSize == 0 {
if cfg.ProtocolOptions.Wumbo() {
cfg.MaxChanSize = int64(MaxBtcFundingAmountWumbo)
} else {
cfg.MaxChanSize = int64(MaxBtcFundingAmount)
}
}
// Ensure that the user specified values for the min and max channel
// size make sense.
if cfg.MaxChanSize < cfg.MinChanSize {
return nil, fmt.Errorf("invalid channel size parameters: "+
"max channel size %v, must be no less than min chan size %v",
cfg.MaxChanSize, cfg.MinChanSize,
)
}
// Don't allow superflous --maxchansize greater than
// BOLT 02 soft-limit for non-wumbo channel
if !cfg.ProtocolOptions.Wumbo() && cfg.MaxChanSize > int64(MaxFundingAmount) {
return nil, fmt.Errorf("invalid channel size parameters: "+
"maximum channel size %v is greater than maximum non-wumbo"+
" channel size %v",
cfg.MaxChanSize, MaxFundingAmount,
)
}
// Ensure a valid max channel fee allocation was set.
if cfg.MaxChannelFeeAllocation <= 0 || cfg.MaxChannelFeeAllocation > 1 {
return nil, fmt.Errorf("invalid max channel fee allocation: "+
@@ -734,12 +863,12 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
// throughout the codebase we required chaincfg.Params. So as a
// temporary hack, we'll mutate the default net params for
// bitcoin with the litecoin specific information.
applyLitecoinParams(&activeNetParams, &ltcParams)
applyLitecoinParams(&cfg.ActiveNetParams, &ltcParams)
switch cfg.Litecoin.Node {
case "ltcd":
err := parseRPCParams(cfg.Litecoin, cfg.LtcdMode,
litecoinChain, funcName)
litecoinChain, funcName, cfg.ActiveNetParams)
if err != nil {
err := fmt.Errorf("unable to load RPC "+
"credentials for ltcd: %v", err)
@@ -751,7 +880,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
"support simnet", funcName)
}
err := parseRPCParams(cfg.Litecoin, cfg.LitecoindMode,
litecoinChain, funcName)
litecoinChain, funcName, cfg.ActiveNetParams)
if err != nil {
err := fmt.Errorf("unable to load RPC "+
"credentials for litecoind: %v", err)
@@ -779,19 +908,19 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
numNets := 0
if cfg.Bitcoin.MainNet {
numNets++
activeNetParams = bitcoinMainNetParams
cfg.ActiveNetParams = bitcoinMainNetParams
}
if cfg.Bitcoin.TestNet3 {
numNets++
activeNetParams = bitcoinTestNetParams
cfg.ActiveNetParams = bitcoinTestNetParams
}
if cfg.Bitcoin.RegTest {
numNets++
activeNetParams = bitcoinRegTestNetParams
cfg.ActiveNetParams = bitcoinRegTestNetParams
}
if cfg.Bitcoin.SimNet {
numNets++
activeNetParams = bitcoinSimNetParams
cfg.ActiveNetParams = bitcoinSimNetParams
}
if numNets > 1 {
str := "%s: The mainnet, testnet, regtest, and " +
@@ -820,6 +949,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
case "btcd":
err := parseRPCParams(
cfg.Bitcoin, cfg.BtcdMode, bitcoinChain, funcName,
cfg.ActiveNetParams,
)
if err != nil {
err := fmt.Errorf("unable to load RPC "+
@@ -834,6 +964,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
err := parseRPCParams(
cfg.Bitcoin, cfg.BitcoindMode, bitcoinChain, funcName,
cfg.ActiveNetParams,
)
if err != nil {
err := fmt.Errorf("unable to load RPC "+
@@ -911,7 +1042,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
cfg.networkDir = filepath.Join(
cfg.DataDir, defaultChainSubDirname,
cfg.registeredChains.PrimaryChain().String(),
normalizeNetwork(activeNetParams.Name),
normalizeNetwork(cfg.ActiveNetParams.Name),
)
// If a custom macaroon directory wasn't specified and the data
@@ -945,7 +1076,7 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
// per network in the same fashion as the data directory.
cfg.LogDir = filepath.Join(cfg.LogDir,
cfg.registeredChains.PrimaryChain().String(),
normalizeNetwork(activeNetParams.Name))
normalizeNetwork(cfg.ActiveNetParams.Name))
// A log writer must be passed in, otherwise we can't function and would
// run into a panic later on.
@@ -1097,12 +1228,30 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
"minbackoff")
}
// Newer versions of lnd added a new sub-config for bolt-specific
// parameters. However we want to also allow existing users to use the
// value on the top-level config. If the outer config value is set,
// then we'll use that directly.
if cfg.SyncFreelist {
cfg.DB.Bolt.SyncFreelist = cfg.SyncFreelist
}
// Ensure that the user hasn't chosen a remote-max-htlc value greater
// than the protocol maximum.
maxRemoteHtlcs := uint16(input.MaxHTLCNumber / 2)
if cfg.DefaultRemoteMaxHtlcs > maxRemoteHtlcs {
return nil, fmt.Errorf("default-remote-max-htlcs (%v) must be "+
"less than %v", cfg.DefaultRemoteMaxHtlcs,
maxRemoteHtlcs)
}
// Validate the subconfigs for workers, caches, and the tower client.
err = lncfg.Validate(
cfg.Workers,
cfg.Caches,
cfg.WtClient,
cfg.DB,
cfg.HealthChecks,
)
if err != nil {
return nil, err
@@ -1125,11 +1274,11 @@ func ValidateConfig(cfg Config, usageMessage string) (*Config, error) {
func (c *Config) localDatabaseDir() string {
return filepath.Join(c.DataDir,
defaultGraphSubDirname,
normalizeNetwork(activeNetParams.Name))
normalizeNetwork(c.ActiveNetParams.Name))
}
func (c *Config) networkName() string {
return normalizeNetwork(activeNetParams.Name)
return normalizeNetwork(c.ActiveNetParams.Name)
}
// CleanAndExpandPath expands environment variables and leading ~ in the
@@ -1159,7 +1308,7 @@ func CleanAndExpandPath(path string) string {
}
func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{}, net chainCode,
funcName string) error { // nolint:unparam
funcName string, netParams bitcoinNetParams) error { // nolint:unparam
// First, we'll check our node config to make sure the RPC parameters
// were set correctly. We'll also determine the path to the conf file
@@ -1269,7 +1418,7 @@ func parseRPCParams(cConfig *lncfg.Chain, nodeConfig interface{}, net chainCode,
case "bitcoind", "litecoind":
nConf := nodeConfig.(*lncfg.Bitcoind)
rpcUser, rpcPass, zmqBlockHost, zmqTxHost, err :=
extractBitcoindRPCParams(confFile)
extractBitcoindRPCParams(netParams.Params.Name, confFile)
if err != nil {
return fmt.Errorf("unable to extract RPC credentials:"+
" %v, cannot start w/o RPC connection",
@@ -1329,13 +1478,13 @@ func extractBtcdRPCParams(btcdConfigPath string) (string, string, error) {
return string(userSubmatches[1]), string(passSubmatches[1]), nil
}
// extractBitcoindParams attempts to extract the RPC credentials for an
// extractBitcoindRPCParams attempts to extract the RPC credentials for an
// existing bitcoind node instance. The passed path is expected to be the
// location of bitcoind's bitcoin.conf on the target system. The routine looks
// for a cookie first, optionally following the datadir configuration option in
// the bitcoin.conf. If it doesn't find one, it looks for rpcuser/rpcpassword.
func extractBitcoindRPCParams(bitcoindConfigPath string) (string, string, string,
string, error) {
func extractBitcoindRPCParams(networkName string,
bitcoindConfigPath string) (string, string, string, string, error) {
// First, we'll open up the bitcoind configuration file found at the
// target destination.
@@ -1393,7 +1542,7 @@ func extractBitcoindRPCParams(bitcoindConfigPath string) (string, string, string
}
chainDir := "/"
switch activeNetParams.Params.Name {
switch networkName {
case "testnet3":
chainDir = "/testnet3/"
case "testnet4":
+23 -29
View File
@@ -10,7 +10,6 @@ import (
"github.com/btcsuite/btcutil"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/sweep"
)
@@ -86,41 +85,36 @@ func (c *anchorResolver) Resolve() (ContractResolver, error) {
// situation. We don't want to force sweep anymore, because the anchor
// lost its special purpose to get the commitment confirmed. It is just
// an output that we want to sweep only if it is economical to do so.
//
// An exclusive group is not necessary anymore, because we know that
// this is the only anchor that can be swept.
//
// We also clear the parent tx information for cpfp, because the
// commitment tx is confirmed.
//
// After a restart or when the remote force closes, the sweeper is not
// yet aware of the anchor. In that case, it will be added as new input
// to the sweeper.
relayFeeRate := c.Sweeper.RelayFeePerKW()
resultChan, err := c.Sweeper.UpdateParams(
c.anchor,
sweep.ParamsUpdate{
anchorInput := input.MakeBaseInput(
&c.anchor,
input.CommitmentAnchor,
&c.anchorSignDescriptor,
c.broadcastHeight,
nil,
)
resultChan, err := c.Sweeper.SweepInput(
&anchorInput,
sweep.Params{
Fee: sweep.FeePreference{
FeeRate: relayFeeRate,
},
Force: false,
},
)
// After a restart or when the remote force closes, the sweeper is not
// yet aware of the anchor. In that case, offer it as a new input to the
// sweeper. An exclusive group is not necessary anymore, because we know
// that this is the only anchor that can be swept.
if err == lnwallet.ErrNotMine {
anchorInput := input.MakeBaseInput(
&c.anchor,
input.CommitmentAnchor,
&c.anchorSignDescriptor,
c.broadcastHeight,
)
resultChan, err = c.Sweeper.SweepInput(
&anchorInput,
sweep.Params{
Fee: sweep.FeePreference{
FeeRate: relayFeeRate,
},
},
)
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
var (
+1 -2
View File
@@ -442,7 +442,6 @@ func (b *boltArbitratorLog) CurrentState() (ArbitratorState, error) {
//
// NOTE: Part of the ContractResolver interface.
func (b *boltArbitratorLog) CommitState(s ArbitratorState) error {
fmt.Printf("yeee: %T\n", b.db)
return kvdb.Batch(b.db, func(tx kvdb.RwTx) error {
scopeBucket, err := tx.CreateTopLevelBucket(b.scopeKey[:])
if err != nil {
@@ -727,7 +726,7 @@ func (b *boltArbitratorLog) FetchContractResolutions() (*ContractResolutions, er
numOutgoing uint32
)
// Next, we'll read out he incoming and outgoing HTLC
// Next, we'll read out the incoming and outgoing HTLC
// resolutions.
err = binary.Read(resReader, endian, &numIncoming)
if err != nil {
+8 -4
View File
@@ -15,6 +15,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/labels"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
"github.com/lightningnetwork/lnd/lnwire"
@@ -59,7 +60,7 @@ type ChainArbitratorConfig struct {
// broadcast our commitment transaction if we have incoming htlcs. This
// value should be set based on our current fee estimation of the
// commitment transaction. We use this to determine when we should
// broadcast instead of the just the HTLC timeout, as we want to ensure
// broadcast instead of just the HTLC timeout, as we want to ensure
// that the commitment transaction is already confirmed, by the time the
// HTLC expires. Otherwise we may end up not settling the htlc on-chain
// because the other party managed to time it out.
@@ -89,7 +90,7 @@ type ChainArbitratorConfig struct {
DeliverResolutionMsg func(...ResolutionMsg) error
// MarkLinkInactive is a function closure that the ChainArbitrator will
// use to mark that active HTLC's shouldn't be attempt ted to be routed
// use to mark that active HTLC's shouldn't be attempted to be routed
// over a particular channel. This function will be called in that a
// ChannelArbitrator decides that it needs to go to chain in order to
// resolve contracts.
@@ -157,7 +158,7 @@ type ChainArbitratorConfig struct {
// resolution.
OnionProcessor OnionProcessor
// PaymentsExpirationGracePeriod indicates is a time window we let the
// PaymentsExpirationGracePeriod indicates a time window we let the
// other node to cancel an outgoing htlc that our node has initiated and
// has timed out.
PaymentsExpirationGracePeriod time.Duration
@@ -715,7 +716,10 @@ func (c *ChainArbitrator) rebroadcast(channel *channeldb.OpenChannel,
log.Infof("Re-publishing %s close tx(%v) for channel %v",
kind, closeTx.TxHash(), chanPoint)
err = c.cfg.PublishTx(closeTx, "")
label := labels.MakeLabel(
labels.LabelTypeChannelClose, &channel.ShortChannelID,
)
err = c.cfg.PublishTx(closeTx, label)
if err != nil && err != lnwallet.ErrDoubleSpend {
log.Warnf("Unable to broadcast %s close tx(%v): %v",
kind, closeTx.TxHash(), err)
+14 -4
View File
@@ -8,8 +8,10 @@ import (
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/chainntnfs"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lnwallet"
)
@@ -80,8 +82,12 @@ func TestChainArbitratorRepublishCloses(t *testing.T) {
published := make(map[chainhash.Hash]int)
chainArbCfg := ChainArbitratorConfig{
ChainIO: &mockChainIO{},
Notifier: &mockNotifier{},
ChainIO: &mock.ChainIO{},
Notifier: &mock.ChainNotifier{
SpendChan: make(chan *chainntnfs.SpendDetail),
EpochChan: make(chan *chainntnfs.BlockEpoch),
ConfChan: make(chan *chainntnfs.TxConfirmation),
},
PublishTx: func(tx *wire.MsgTx, _ string) error {
published[tx.TxHash()]++
return nil
@@ -172,8 +178,12 @@ func TestResolveContract(t *testing.T) {
// chain arbitrator that should pick up these new channels and launch
// resolver for them.
chainArbCfg := ChainArbitratorConfig{
ChainIO: &mockChainIO{},
Notifier: &mockNotifier{},
ChainIO: &mock.ChainIO{},
Notifier: &mock.ChainNotifier{
SpendChan: make(chan *chainntnfs.SpendDetail),
EpochChan: make(chan *chainntnfs.BlockEpoch),
ConfChan: make(chan *chainntnfs.TxConfirmation),
},
PublishTx: func(tx *wire.MsgTx, _ string) error {
return nil
},
+2 -2
View File
@@ -500,7 +500,7 @@ func (c *chainWatcher) closeObserver(spendNtfn *chainntnfs.SpendEvent) {
select {
// We've detected a spend of the channel onchain! Depending on the type
// of spend, we'll act accordingly , so we'll examine the spending
// of spend, we'll act accordingly, so we'll examine the spending
// transaction to determine what we should do.
//
// TODO(Roasbeef): need to be able to ensure this only triggers
@@ -568,7 +568,7 @@ func (c *chainWatcher) closeObserver(spendNtfn *chainntnfs.SpendEvent) {
// Next, we'll check to see if this is a cooperative channel
// closure or not. This is characterized by having an input
// sequence number that's finalized. This won't happen with
// sequence number that's finalized. This won't happen with
// regular commitment transactions due to the state hint
// encoding scheme.
if commitTxBroadcast.TxIn[0].Sequence == wire.MaxTxInSequenceNum {
+21 -56
View File
@@ -7,58 +7,15 @@ import (
"testing"
"time"
"github.com/btcsuite/btcd/chaincfg/chainhash"
"github.com/btcsuite/btcd/wire"
"github.com/lightningnetwork/lnd/chainntnfs"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
)
type mockNotifier struct {
spendChan chan *chainntnfs.SpendDetail
epochChan chan *chainntnfs.BlockEpoch
confChan chan *chainntnfs.TxConfirmation
}
func (m *mockNotifier) RegisterConfirmationsNtfn(txid *chainhash.Hash, _ []byte, numConfs,
heightHint uint32) (*chainntnfs.ConfirmationEvent, error) {
return &chainntnfs.ConfirmationEvent{
Confirmed: m.confChan,
Cancel: func() {},
}, nil
}
func (m *mockNotifier) RegisterBlockEpochNtfn(
bestBlock *chainntnfs.BlockEpoch) (*chainntnfs.BlockEpochEvent, error) {
return &chainntnfs.BlockEpochEvent{
Epochs: m.epochChan,
Cancel: func() {},
}, nil
}
func (m *mockNotifier) Start() error {
return nil
}
func (m *mockNotifier) Started() bool {
return true
}
func (m *mockNotifier) Stop() error {
return nil
}
func (m *mockNotifier) RegisterSpendNtfn(outpoint *wire.OutPoint, _ []byte,
heightHint uint32) (*chainntnfs.SpendEvent, error) {
return &chainntnfs.SpendEvent{
Spend: m.spendChan,
Cancel: func() {},
}, nil
}
// TestChainWatcherRemoteUnilateralClose tests that the chain watcher is able
// to properly detect a normal unilateral close by the remote node using their
// lowest commitment.
@@ -77,8 +34,10 @@ func TestChainWatcherRemoteUnilateralClose(t *testing.T) {
// With the channels created, we'll now create a chain watcher instance
// which will be watching for any closes of Alice's channel.
aliceNotifier := &mockNotifier{
spendChan: make(chan *chainntnfs.SpendDetail),
aliceNotifier := &mock.ChainNotifier{
SpendChan: make(chan *chainntnfs.SpendDetail),
EpochChan: make(chan *chainntnfs.BlockEpoch),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
aliceChainWatcher, err := newChainWatcher(chainWatcherConfig{
chanState: aliceChannel.State(),
@@ -107,7 +66,7 @@ func TestChainWatcherRemoteUnilateralClose(t *testing.T) {
SpenderTxHash: &bobTxHash,
SpendingTx: bobCommit,
}
aliceNotifier.spendChan <- bobSpend
aliceNotifier.SpendChan <- bobSpend
// We should get a new spend event over the remote unilateral close
// event channel.
@@ -166,8 +125,10 @@ func TestChainWatcherRemoteUnilateralClosePendingCommit(t *testing.T) {
// With the channels created, we'll now create a chain watcher instance
// which will be watching for any closes of Alice's channel.
aliceNotifier := &mockNotifier{
spendChan: make(chan *chainntnfs.SpendDetail),
aliceNotifier := &mock.ChainNotifier{
SpendChan: make(chan *chainntnfs.SpendDetail),
EpochChan: make(chan *chainntnfs.BlockEpoch),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
aliceChainWatcher, err := newChainWatcher(chainWatcherConfig{
chanState: aliceChannel.State(),
@@ -216,7 +177,7 @@ func TestChainWatcherRemoteUnilateralClosePendingCommit(t *testing.T) {
SpenderTxHash: &bobTxHash,
SpendingTx: bobCommit,
}
aliceNotifier.spendChan <- bobSpend
aliceNotifier.SpendChan <- bobSpend
// We should get a new spend event over the remote unilateral close
// event channel.
@@ -292,8 +253,10 @@ func TestChainWatcherDataLossProtect(t *testing.T) {
// With the channels created, we'll now create a chain watcher
// instance which will be watching for any closes of Alice's
// channel.
aliceNotifier := &mockNotifier{
spendChan: make(chan *chainntnfs.SpendDetail),
aliceNotifier := &mock.ChainNotifier{
SpendChan: make(chan *chainntnfs.SpendDetail),
EpochChan: make(chan *chainntnfs.BlockEpoch),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
aliceChainWatcher, err := newChainWatcher(chainWatcherConfig{
chanState: aliceChannel.State(),
@@ -350,7 +313,7 @@ func TestChainWatcherDataLossProtect(t *testing.T) {
SpenderTxHash: &bobTxHash,
SpendingTx: bobCommit,
}
aliceNotifier.spendChan <- bobSpend
aliceNotifier.SpendChan <- bobSpend
// We should get a new uni close resolution that indicates we
// processed the DLP scenario.
@@ -461,8 +424,10 @@ func TestChainWatcherLocalForceCloseDetect(t *testing.T) {
// With the channels created, we'll now create a chain watcher
// instance which will be watching for any closes of Alice's
// channel.
aliceNotifier := &mockNotifier{
spendChan: make(chan *chainntnfs.SpendDetail),
aliceNotifier := &mock.ChainNotifier{
SpendChan: make(chan *chainntnfs.SpendDetail),
EpochChan: make(chan *chainntnfs.BlockEpoch),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
aliceChainWatcher, err := newChainWatcher(chainWatcherConfig{
chanState: aliceChannel.State(),
@@ -517,7 +482,7 @@ func TestChainWatcherLocalForceCloseDetect(t *testing.T) {
SpenderTxHash: &aliceTxHash,
SpendingTx: aliceCommit,
}
aliceNotifier.spendChan <- aliceSpend
aliceNotifier.SpendChan <- aliceSpend
// We should get a local force close event from Alice as she
// should be able to detect the close based on the commitment
+27 -12
View File
@@ -16,6 +16,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/labels"
"github.com/lightningnetwork/lnd/lntypes"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
@@ -29,6 +30,12 @@ var (
"process of being force closed")
)
const (
// anchorSweepConfTarget is the conf target used when sweeping
// commitment anchors.
anchorSweepConfTarget = 6
)
// WitnessSubscription represents an intent to be notified once new witnesses
// are discovered by various active contract resolvers. A contract resolver may
// use this to be notified of when it can satisfy an incoming contract after we
@@ -874,7 +881,11 @@ func (c *ChannelArbitrator) stateStep(
// At this point, we'll now broadcast the commitment
// transaction itself.
if err := c.cfg.PublishTx(closeTx, ""); err != nil {
label := labels.MakeLabel(
labels.LabelTypeChannelClose, &c.cfg.ShortChanID,
)
if err := c.cfg.PublishTx(closeTx, label); err != nil {
log.Errorf("ChannelArbitrator(%v): unable to broadcast "+
"close tx: %v", c.cfg.ChanPoint, err)
if err != lnwallet.ErrDoubleSpend {
@@ -1055,9 +1066,6 @@ func (c *ChannelArbitrator) sweepAnchors(anchors []*lnwallet.AnchorResolution,
// anchors from being batched together.
exclusiveGroup := c.cfg.ShortChanID.ToUint64()
// Retrieve the current minimum fee rate from the sweeper.
minFeeRate := c.cfg.Sweeper.RelayFeePerKW()
for _, anchor := range anchors {
log.Debugf("ChannelArbitrator(%v): pre-confirmation sweep of "+
"anchor of tx %v", c.cfg.ChanPoint, anchor.CommitAnchor)
@@ -1068,18 +1076,25 @@ func (c *ChannelArbitrator) sweepAnchors(anchors []*lnwallet.AnchorResolution,
input.CommitmentAnchor,
&anchor.AnchorSignDescriptor,
heightHint,
&input.TxInfo{
Fee: anchor.CommitFee,
Weight: anchor.CommitWeight,
},
)
// Sweep anchor output with the minimum fee rate. This usually
// (up to a min relay fee of 3 sat/b) means that the anchor
// sweep will be economical. Also signal that this is a force
// sweep. If the user decides to bump the fee on the anchor
// sweep, it will be swept even if it isn't economical.
// Sweep anchor output with a confirmation target fee
// preference. Because this is a cpfp-operation, the anchor will
// only be attempted to sweep when the current fee estimate for
// the confirmation target exceeds the commit fee rate.
//
// Also signal that this is a force sweep, so that the anchor
// will be swept even if it isn't economical purely based on the
// anchor value.
_, err := c.cfg.Sweeper.SweepInput(
&anchorInput,
sweep.Params{
Fee: sweep.FeePreference{
FeeRate: minFeeRate,
ConfTarget: anchorSweepConfTarget,
},
Force: true,
ExclusiveGroup: &exclusiveGroup,
@@ -1987,7 +2002,7 @@ func (c *ChannelArbitrator) resolveContract(currentContract ContractResolver) {
switch {
// If this contract produced another, then this means
// the current contract was only able to be partially
// resolved in this step. So we'll not a contract swap
// resolved in this step. So we'll do a contract swap
// within our logs: the new contract will take the
// place of the old one.
case nextContract != nil:
@@ -2082,7 +2097,7 @@ func (c *ChannelArbitrator) UpdateContractSignals(newSignals *ContractSignals) {
// channelAttendant is the primary goroutine that acts at the judicial
// arbitrator between our channel state, the remote channel peer, and the
// blockchain Our judge). This goroutine will ensure that we faithfully execute
// blockchain (Our judge). This goroutine will ensure that we faithfully execute
// all clauses of our contract in the case that we need to go on-chain for a
// dispute. Currently, two such conditions warrant our intervention: when an
// outgoing HTLC is about to timeout, and when we know the pre-image for an
+11 -10
View File
@@ -19,6 +19,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/clock"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
)
@@ -334,10 +335,10 @@ func createTestChannelArbitrator(t *testing.T, log ArbitratorLog,
},
OutgoingBroadcastDelta: 5,
IncomingBroadcastDelta: 5,
Notifier: &mockNotifier{
epochChan: make(chan *chainntnfs.BlockEpoch),
spendChan: make(chan *chainntnfs.SpendDetail),
confChan: make(chan *chainntnfs.TxConfirmation),
Notifier: &mock.ChainNotifier{
EpochChan: make(chan *chainntnfs.BlockEpoch),
SpendChan: make(chan *chainntnfs.SpendDetail),
ConfChan: make(chan *chainntnfs.TxConfirmation),
},
IncubateOutputs: func(wire.OutPoint,
*lnwallet.OutgoingHtlcResolution,
@@ -878,7 +879,7 @@ func TestChannelArbitratorLocalForceClosePendingHtlc(t *testing.T) {
// We'll grab the old notifier here as our resolvers are still holding
// a reference to this instance, and a new one will be created when we
// restart the channel arb below.
oldNotifier := chanArb.cfg.Notifier.(*mockNotifier)
oldNotifier := chanArb.cfg.Notifier.(*mock.ChainNotifier)
// At this point, in order to simulate a restart, we'll re-create the
// channel arbitrator. We do this to ensure that all information
@@ -927,7 +928,7 @@ func TestChannelArbitratorLocalForceClosePendingHtlc(t *testing.T) {
}
// Send a notification that the expiry height has been reached.
oldNotifier.epochChan <- &chainntnfs.BlockEpoch{Height: 10}
oldNotifier.EpochChan <- &chainntnfs.BlockEpoch{Height: 10}
// htlcOutgoingContestResolver is now transforming into a
// htlcTimeoutResolver and should send the contract off for incubation.
@@ -939,7 +940,7 @@ func TestChannelArbitratorLocalForceClosePendingHtlc(t *testing.T) {
// Notify resolver that the HTLC output of the commitment has been
// spent.
oldNotifier.spendChan <- &chainntnfs.SpendDetail{SpendingTx: closeTx}
oldNotifier.SpendChan <- &chainntnfs.SpendDetail{SpendingTx: closeTx}
// Finally, we should also receive a resolution message instructing the
// switch to cancel back the HTLC.
@@ -967,7 +968,7 @@ func TestChannelArbitratorLocalForceClosePendingHtlc(t *testing.T) {
}
// Notify resolver that the second level transaction is spent.
oldNotifier.spendChan <- &chainntnfs.SpendDetail{SpendingTx: closeTx}
oldNotifier.SpendChan <- &chainntnfs.SpendDetail{SpendingTx: closeTx}
// At this point channel should be marked as resolved.
chanArbCtxNew.AssertStateTransitions(StateFullyResolved)
@@ -2236,9 +2237,9 @@ func TestChannelArbitratorAnchors(t *testing.T) {
t.Fatalf("expected anchor resolver, got %T", resolver)
}
// The anchor resolver is expected to offer the anchor input to the
// The anchor resolver is expected to re-offer the anchor input to the
// sweeper.
<-chanArbCtx.sweeper.updatedInputs
<-chanArbCtx.sweeper.sweptInputs
// The mock sweeper immediately signals success for that input. This
// should transition the channel to the resolved state.
+9 -8
View File
@@ -10,6 +10,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
"github.com/lightningnetwork/lnd/sweep"
@@ -17,7 +18,7 @@ import (
type commitSweepResolverTestContext struct {
resolver *commitSweepResolver
notifier *mockNotifier
notifier *mock.ChainNotifier
sweeper *mockSweeper
resolverResultChan chan resolveResult
t *testing.T
@@ -26,10 +27,10 @@ type commitSweepResolverTestContext struct {
func newCommitSweepResolverTestContext(t *testing.T,
resolution *lnwallet.CommitOutputResolution) *commitSweepResolverTestContext {
notifier := &mockNotifier{
epochChan: make(chan *chainntnfs.BlockEpoch),
spendChan: make(chan *chainntnfs.SpendDetail),
confChan: make(chan *chainntnfs.TxConfirmation),
notifier := &mock.ChainNotifier{
EpochChan: make(chan *chainntnfs.BlockEpoch),
SpendChan: make(chan *chainntnfs.SpendDetail),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
sweeper := newMockSweeper()
@@ -83,7 +84,7 @@ func (i *commitSweepResolverTestContext) resolve() {
}
func (i *commitSweepResolverTestContext) notifyEpoch(height int32) {
i.notifier.epochChan <- &chainntnfs.BlockEpoch{
i.notifier.EpochChan <- &chainntnfs.BlockEpoch{
Height: height,
}
}
@@ -189,7 +190,7 @@ func TestCommitSweepResolverNoDelay(t *testing.T) {
spendTx := &wire.MsgTx{}
spendHash := spendTx.TxHash()
ctx.notifier.confChan <- &chainntnfs.TxConfirmation{
ctx.notifier.ConfChan <- &chainntnfs.TxConfirmation{
Tx: spendTx,
}
@@ -267,7 +268,7 @@ func testCommitSweepResolverDelay(t *testing.T, sweepErr error) {
ctx.resolve()
ctx.notifier.confChan <- &chainntnfs.TxConfirmation{
ctx.notifier.ConfChan <- &chainntnfs.TxConfirmation{
BlockHeight: testInitialBlockHeight - 1,
}
+7 -6
View File
@@ -12,6 +12,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/htlcswitch/hop"
"github.com/lightningnetwork/lnd/invoices"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lntypes"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
@@ -295,7 +296,7 @@ type incomingResolverTestContext struct {
registry *mockRegistry
witnessBeacon *mockWitnessBeacon
resolver *htlcIncomingContestResolver
notifier *mockNotifier
notifier *mock.ChainNotifier
onionProcessor *mockOnionProcessor
resolveErr chan error
nextResolver ContractResolver
@@ -303,10 +304,10 @@ type incomingResolverTestContext struct {
}
func newIncomingResolverTestContext(t *testing.T, isExit bool) *incomingResolverTestContext {
notifier := &mockNotifier{
epochChan: make(chan *chainntnfs.BlockEpoch),
spendChan: make(chan *chainntnfs.SpendDetail),
confChan: make(chan *chainntnfs.TxConfirmation),
notifier := &mock.ChainNotifier{
EpochChan: make(chan *chainntnfs.BlockEpoch),
SpendChan: make(chan *chainntnfs.SpendDetail),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
witnessBeacon := newMockWitnessBeacon()
registry := &mockRegistry{
@@ -377,7 +378,7 @@ func (i *incomingResolverTestContext) resolve() {
}
func (i *incomingResolverTestContext) notifyEpoch(height int32) {
i.notifier.epochChan <- &chainntnfs.BlockEpoch{
i.notifier.EpochChan <- &chainntnfs.BlockEpoch{
Height: height,
}
}
@@ -9,6 +9,7 @@ import (
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lntypes"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
@@ -80,7 +81,7 @@ func TestHtlcOutgoingResolverRemoteClaim(t *testing.T) {
spendHash := spendTx.TxHash()
ctx.notifier.spendChan <- &chainntnfs.SpendDetail{
ctx.notifier.SpendChan <- &chainntnfs.SpendDetail{
SpendingTx: spendTx,
SpenderTxHash: &spendHash,
}
@@ -114,7 +115,7 @@ type resolveResult struct {
type outgoingResolverTestContext struct {
resolver *htlcOutgoingContestResolver
notifier *mockNotifier
notifier *mock.ChainNotifier
preimageDB *mockWitnessBeacon
resolverResultChan chan resolveResult
resolutionChan chan ResolutionMsg
@@ -122,10 +123,10 @@ type outgoingResolverTestContext struct {
}
func newOutgoingResolverTestContext(t *testing.T) *outgoingResolverTestContext {
notifier := &mockNotifier{
epochChan: make(chan *chainntnfs.BlockEpoch),
spendChan: make(chan *chainntnfs.SpendDetail),
confChan: make(chan *chainntnfs.TxConfirmation),
notifier := &mock.ChainNotifier{
EpochChan: make(chan *chainntnfs.BlockEpoch),
SpendChan: make(chan *chainntnfs.SpendDetail),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
checkPointChan := make(chan struct{}, 1)
@@ -212,7 +213,7 @@ func (i *outgoingResolverTestContext) resolve() {
}
func (i *outgoingResolverTestContext) notifyEpoch(height int32) {
i.notifier.epochChan <- &chainntnfs.BlockEpoch{
i.notifier.EpochChan <- &chainntnfs.BlockEpoch{
Height: height,
}
}
+9 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/labels"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/sweep"
)
@@ -157,7 +158,10 @@ func (h *htlcSuccessResolver) Resolve() (ContractResolver, error) {
// Regardless of whether an existing transaction was found or newly
// constructed, we'll broadcast the sweep transaction to the
// network.
err := h.PublishTx(h.sweepTx, "")
label := labels.MakeLabel(
labels.LabelTypeChannelClose, &h.ShortChanID,
)
err := h.PublishTx(h.sweepTx, label)
if err != nil {
log.Infof("%T(%x): unable to publish tx: %v",
h, h.htlc.RHash[:], err)
@@ -206,7 +210,10 @@ func (h *htlcSuccessResolver) Resolve() (ContractResolver, error) {
// the claiming process.
//
// TODO(roasbeef): after changing sighashes send to tx bundler
err := h.PublishTx(h.htlcResolution.SignedSuccessTx, "")
label := labels.MakeLabel(
labels.LabelTypeChannelClose, &h.ShortChanID,
)
err := h.PublishTx(h.htlcResolution.SignedSuccessTx, label)
if err != nil {
return nil, err
}
+8 -7
View File
@@ -8,6 +8,7 @@ import (
"github.com/lightningnetwork/lnd/chainntnfs"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwire"
)
@@ -16,16 +17,16 @@ var testHtlcAmt = lnwire.MilliSatoshi(200000)
type htlcSuccessResolverTestContext struct {
resolver *htlcSuccessResolver
notifier *mockNotifier
notifier *mock.ChainNotifier
resolverResultChan chan resolveResult
t *testing.T
}
func newHtlcSuccessResolverTextContext(t *testing.T) *htlcSuccessResolverTestContext {
notifier := &mockNotifier{
epochChan: make(chan *chainntnfs.BlockEpoch),
spendChan: make(chan *chainntnfs.SpendDetail),
confChan: make(chan *chainntnfs.TxConfirmation),
notifier := &mock.ChainNotifier{
EpochChan: make(chan *chainntnfs.BlockEpoch),
SpendChan: make(chan *chainntnfs.SpendDetail),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
checkPointChan := make(chan struct{}, 1)
@@ -116,7 +117,7 @@ func TestSingleStageSuccess(t *testing.T) {
// We send a confirmation for our sweep tx to indicate that our sweep
// succeeded.
resolve := func(ctx *htlcSuccessResolverTestContext) {
ctx.notifier.confChan <- &chainntnfs.TxConfirmation{
ctx.notifier.ConfChan <- &chainntnfs.TxConfirmation{
Tx: ctx.resolver.sweepTx,
BlockHeight: testInitialBlockHeight - 1,
}
@@ -165,7 +166,7 @@ func TestSecondStageResolution(t *testing.T) {
// We send a spend notification for our output to resolve our htlc.
resolve := func(ctx *htlcSuccessResolverTestContext) {
ctx.notifier.spendChan <- &chainntnfs.SpendDetail{
ctx.notifier.SpendChan <- &chainntnfs.SpendDetail{
SpendingTx: sweepTx,
SpenderTxHash: &sweepHash,
}
+10 -33
View File
@@ -7,7 +7,6 @@ import (
"testing"
"time"
"github.com/btcsuite/btcd/btcec"
"github.com/btcsuite/btcd/txscript"
"github.com/btcsuite/btcd/wire"
"github.com/btcsuite/btcutil"
@@ -15,33 +14,11 @@ import (
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lntypes"
"github.com/lightningnetwork/lnd/lnwallet"
)
type dummySignature struct{}
func (s *dummySignature) Serialize() []byte {
return []byte{}
}
func (s *dummySignature) Verify(_ []byte, _ *btcec.PublicKey) bool {
return true
}
type mockSigner struct {
}
func (m *mockSigner) SignOutputRaw(tx *wire.MsgTx,
signDesc *input.SignDescriptor) (input.Signature, error) {
return &dummySignature{}, nil
}
func (m *mockSigner) ComputeInputScript(tx *wire.MsgTx,
signDesc *input.SignDescriptor) (*input.Script, error) {
return nil, nil
}
type mockWitnessBeacon struct {
preImageUpdates chan lntypes.Preimage
newPreimages chan []lntypes.Preimage
@@ -93,7 +70,7 @@ func TestHtlcTimeoutResolver(t *testing.T) {
copy(fakePreimage[:], fakePreimageBytes)
signer := &mockSigner{}
signer := &mock.DummySigner{}
sweepTx := &wire.MsgTx{
TxIn: []*wire.TxIn{
{
@@ -164,7 +141,7 @@ func TestHtlcTimeoutResolver(t *testing.T) {
timeout: true,
txToBroadcast: func() (*wire.MsgTx, error) {
witness, err := input.SenderHtlcSpendTimeout(
&dummySignature{}, txscript.SigHashAll,
&mock.DummySignature{}, txscript.SigHashAll,
signer, fakeSignDesc, sweepTx,
)
if err != nil {
@@ -189,7 +166,7 @@ func TestHtlcTimeoutResolver(t *testing.T) {
timeout: false,
txToBroadcast: func() (*wire.MsgTx, error) {
witness, err := input.ReceiverHtlcSpendRedeem(
&dummySignature{}, txscript.SigHashAll,
&mock.DummySignature{}, txscript.SigHashAll,
fakePreimageBytes, signer, fakeSignDesc,
sweepTx,
)
@@ -226,10 +203,10 @@ func TestHtlcTimeoutResolver(t *testing.T) {
},
}
notifier := &mockNotifier{
epochChan: make(chan *chainntnfs.BlockEpoch),
spendChan: make(chan *chainntnfs.SpendDetail),
confChan: make(chan *chainntnfs.TxConfirmation),
notifier := &mock.ChainNotifier{
EpochChan: make(chan *chainntnfs.BlockEpoch),
SpendChan: make(chan *chainntnfs.SpendDetail),
ConfChan: make(chan *chainntnfs.TxConfirmation),
}
witnessBeacon := newMockWitnessBeacon()
@@ -354,7 +331,7 @@ func TestHtlcTimeoutResolver(t *testing.T) {
spendTxHash := spendingTx.TxHash()
select {
case notifier.spendChan <- &chainntnfs.SpendDetail{
case notifier.SpendChan <- &chainntnfs.SpendDetail{
SpendingTx: spendingTx,
SpenderTxHash: &spendTxHash,
}:
@@ -411,7 +388,7 @@ func TestHtlcTimeoutResolver(t *testing.T) {
// only if this is a local commitment transaction.
if !testCase.remoteCommit {
select {
case notifier.spendChan <- &chainntnfs.SpendDetail{
case notifier.SpendChan <- &chainntnfs.SpendDetail{
SpendingTx: spendingTx,
SpenderTxHash: &spendTxHash,
}:
+12 -4
View File
@@ -287,6 +287,10 @@ type DNSSeedBootstrapper struct {
// the network seed.
dnsSeeds [][2]string
net tor.Net
// timeout is the maximum amount of time a dial will wait for a connect to
// complete.
timeout time.Duration
}
// A compile time assertion to ensure that DNSSeedBootstrapper meets the
@@ -300,8 +304,10 @@ var _ NetworkPeerBootstrapper = (*ChannelGraphBootstrapper)(nil)
// used as a fallback for manual TCP resolution in the case of an error
// receiving the UDP response. The second host should return a single A record
// with the IP address of the authoritative name server.
func NewDNSSeedBootstrapper(seeds [][2]string, net tor.Net) NetworkPeerBootstrapper {
return &DNSSeedBootstrapper{dnsSeeds: seeds, net: net}
func NewDNSSeedBootstrapper(
seeds [][2]string, net tor.Net,
timeout time.Duration) NetworkPeerBootstrapper {
return &DNSSeedBootstrapper{dnsSeeds: seeds, net: net, timeout: timeout}
}
// fallBackSRVLookup attempts to manually query for SRV records we need to
@@ -327,7 +333,7 @@ func (d *DNSSeedBootstrapper) fallBackSRVLookup(soaShim string,
// Once we have the IP address, we'll establish a TCP connection using
// port 53.
dnsServer := net.JoinHostPort(addrs[0], "53")
conn, err := d.net.Dial("tcp", dnsServer)
conn, err := d.net.Dial("tcp", dnsServer, d.timeout)
if err != nil {
return nil, err
}
@@ -389,7 +395,9 @@ search:
// obtain a random sample of the encoded public keys of nodes.
// We use the lndLookupSRV function for this task.
primarySeed := dnsSeedTuple[0]
_, addrs, err := d.net.LookupSRV("nodes", "tcp", primarySeed)
_, addrs, err := d.net.LookupSRV(
"nodes", "tcp", primarySeed, d.timeout,
)
if err != nil {
log.Tracef("Unable to lookup SRV records via "+
"primary seed (%v): %v", primarySeed, err)
+8 -28
View File
@@ -23,8 +23,8 @@ import (
"github.com/go-errors/errors"
"github.com/lightningnetwork/lnd/chainntnfs"
"github.com/lightningnetwork/lnd/channeldb"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lnpeer"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lntest/wait"
"github.com/lightningnetwork/lnd/lnwire"
"github.com/lightningnetwork/lnd/netann"
@@ -92,26 +92,6 @@ func makeTestDB() (*channeldb.DB, func(), error) {
return cdb, cleanUp, nil
}
type mockSigner struct {
privKey *btcec.PrivateKey
}
func (n *mockSigner) SignMessage(pubKey *btcec.PublicKey,
msg []byte) (input.Signature, error) {
if !pubKey.IsEqual(n.privKey.PubKey()) {
return nil, fmt.Errorf("unknown public key")
}
digest := chainhash.DoubleHashB(msg)
sign, err := n.privKey.Sign(digest)
if err != nil {
return nil, fmt.Errorf("can't sign the message: %v", err)
}
return sign, nil
}
type mockGraphSource struct {
bestHeight uint32
@@ -555,7 +535,7 @@ func createNodeAnnouncement(priv *btcec.PrivateKey,
a.ExtraOpaqueData = extraBytes[0]
}
signer := mockSigner{priv}
signer := mock.SingleSigner{Privkey: priv}
sig, err := netann.SignAnnouncement(&signer, priv.PubKey(), a)
if err != nil {
return nil, err
@@ -607,7 +587,7 @@ func createUpdateAnnouncement(blockHeight uint32,
func signUpdate(nodeKey *btcec.PrivateKey, a *lnwire.ChannelUpdate) error {
pub := nodeKey.PubKey()
signer := mockSigner{nodeKey}
signer := mock.SingleSigner{Privkey: nodeKey}
sig, err := netann.SignAnnouncement(&signer, pub, a)
if err != nil {
return err
@@ -649,7 +629,7 @@ func createRemoteChannelAnnouncement(blockHeight uint32,
a := createAnnouncementWithoutProof(blockHeight, extraBytes...)
pub := nodeKeyPriv1.PubKey()
signer := mockSigner{nodeKeyPriv1}
signer := mock.SingleSigner{Privkey: nodeKeyPriv1}
sig, err := netann.SignAnnouncement(&signer, pub, a)
if err != nil {
return nil, err
@@ -660,7 +640,7 @@ func createRemoteChannelAnnouncement(blockHeight uint32,
}
pub = nodeKeyPriv2.PubKey()
signer = mockSigner{nodeKeyPriv2}
signer = mock.SingleSigner{Privkey: nodeKeyPriv2}
sig, err = netann.SignAnnouncement(&signer, pub, a)
if err != nil {
return nil, err
@@ -671,7 +651,7 @@ func createRemoteChannelAnnouncement(blockHeight uint32,
}
pub = bitcoinKeyPriv1.PubKey()
signer = mockSigner{bitcoinKeyPriv1}
signer = mock.SingleSigner{Privkey: bitcoinKeyPriv1}
sig, err = netann.SignAnnouncement(&signer, pub, a)
if err != nil {
return nil, err
@@ -682,7 +662,7 @@ func createRemoteChannelAnnouncement(blockHeight uint32,
}
pub = bitcoinKeyPriv2.PubKey()
signer = mockSigner{bitcoinKeyPriv2}
signer = mock.SingleSigner{Privkey: bitcoinKeyPriv2}
sig, err = netann.SignAnnouncement(&signer, pub, a)
if err != nil {
return nil, err
@@ -761,7 +741,7 @@ func createTestCtx(startHeight uint32) (*testCtx, func(), error) {
RotateTicker: ticker.NewForce(DefaultSyncerRotationInterval),
HistoricalSyncTicker: ticker.NewForce(DefaultHistoricalSyncInterval),
NumActiveSyncers: 3,
AnnSigner: &mockSigner{nodeKeyPriv1},
AnnSigner: &mock.SingleSigner{Privkey: nodeKeyPriv1},
SubBatchDelay: time.Second * 5,
MinimumBatchSize: 10,
}, nodeKeyPub1)
+3 -3
View File
@@ -29,7 +29,7 @@
**Note**: The minimum version of Go supported is Go 1.13. We recommend that
users use the latest version of Go, which at the time of writing is
[`1.13`](https://blog.golang.org/go1.13).
[`1.15`](https://blog.golang.org/go1.15).
On Linux:
@@ -95,7 +95,7 @@
* **Go modules:** This project uses [Go modules](https://github.com/golang/go/wiki/Modules)
to manage dependencies as well as to provide *reproducible builds*.
Usage of Go modules (with Go 1.12) means that you no longer need to clone
Usage of Go modules (with Go 1.13) means that you no longer need to clone
`lnd` into your `$GOPATH` for development purposes. Instead, your `lnd`
repo can now live anywhere!
@@ -124,7 +124,7 @@ make install
**NOTE**: Our instructions still use the `$GOPATH` directory from prior
versions of Go, but with Go 1.12, it's now possible for `lnd` to live
versions of Go, but with Go 1.13, it's now possible for `lnd` to live
_anywhere_ on your file system.
For Windows WSL users, make will need to be referenced directly via
+8 -1
View File
@@ -81,7 +81,14 @@ methods. This means a few important things:
You can also run `lnd` with the `--no-macaroons` option, which skips the
creation of the macaroon files and all macaroon checks within the RPC server.
This means you can still pass a macaroon to the RPC server with a client, but
it won't be checked for validity.
it won't be checked for validity. Note that disabling authentication of a server
that's listening on a public interface is not allowed. This means the
`--no-macaroons` option is only permitted when the RPC server is in a private
network. In CIDR notation, the following IPs are considered private,
- [`169.254.0.0/16` and `fe80::/10`](https://en.wikipedia.org/wiki/Link-local_address).
- [`224.0.0.0/4` and `ff00::/8`](https://en.wikipedia.org/wiki/Multicast_address).
- [`10.0.0.0/8`, `172.16.0.0/12` and `192.168.0.0/16`](https://tools.ietf.org/html/rfc1918).
- [`fc00::/7`](https://tools.ietf.org/html/rfc4193).
Since `lnd` requires macaroons by default in order to call RPC methods, `lncli`
now reads a macaroon and provides it in the RPC call. Unless the path is
+99
View File
@@ -0,0 +1,99 @@
# WebSockets with `lnd`'s REST API
This document describes how streaming response REST calls can be used correctly
by making use of the WebSocket API.
As an example, we are going to write a simple JavaScript program that subscribes
to `lnd`'s
[block notification RPC](https://api.lightning.community/#v2-chainnotifier-register-blocks).
The WebSocket will be kept open as long as `lnd` runs and JavaScript program
isn't stopped.
## Browser environment
When using WebSockets in a browser, there are certain security limitations of
what header fields are allowed to be sent. Therefore, the macaroon cannot just
be added as a `Grpc-Metadata-Macaroon` header field as it would work with normal
REST calls. The browser will just ignore that header field and not send it.
Instead we have added a workaround in `lnd`'s WebSocket proxy that allows
sending the macaroon as a WebSocket "protocol":
```javascript
const host = 'localhost:8080'; // The default REST port of lnd, can be overwritten with --restlisten=ip:port
const macaroon = '0201036c6e6402eb01030a10625e7e60fd00f5a6f9cd53f33fc82a...'; // The hex encoded macaroon to send
const initialRequest = { // The initial request to send (see API docs for each RPC).
hash: "xlkMdV382uNPskw6eEjDGFMQHxHNnZZgL47aVDSwiRQ=", // Just some example to show that all `byte` fields always have to be base64 encoded in the REST API.
height: 144,
}
// The protocol is our workaround for sending the macaroon because custom header
// fields aren't allowed to be sent by the browser when opening a WebSocket.
const protocolString = 'Grpc-Metadata-Macaroon+' + macaroon;
// Let's now connect the web socket. Notice that all WebSocket open calls are
// always GET requests. If the RPC expects a call to be POST or DELETE (see API
// docs to find out), the query parameter "method" can be set to overwrite.
const wsUrl = 'wss://' + host + '/v2/chainnotifier/register/blocks?method=POST';
let ws = new WebSocket(wsUrl, protocolString);
ws.onopen = function (event) {
// After the WS connection is establishes, lnd expects the client to send the
// initial message. If an RPC doesn't have any request parameters, an empty
// JSON object has to be sent as a string, for example: ws.send('{}')
ws.send(JSON.stringify(initialRequest));
}
ws.onmessage = function (event) {
// We received a new message.
console.log(event);
// The data we're really interested in is in data and is always a string
// that needs to be parsed as JSON and always contains a "result" field:
console.log("Payload: ");
console.log(JSON.parse(event.data).result);
}
ws.onerror = function (event) {
// An error occured, let's log it to the console.
console.log(event);
}
```
## Node.js environment
With Node.js it is a bit easier to use the streaming response APIs because we
can set the macaroon header field directly. This is the example from the API
docs:
```javascript
// --------------------------
// Example with websockets:
// --------------------------
const WebSocket = require('ws');
const fs = require('fs');
const macaroon = fs.readFileSync('LND_DIR/data/chain/bitcoin/simnet/admin.macaroon').toString('hex');
let ws = new WebSocket('wss://localhost:8080/v2/chainnotifier/register/blocks?method=POST', {
// Work-around for self-signed certificates.
rejectUnauthorized: false,
headers: {
'Grpc-Metadata-Macaroon': macaroon,
},
});
let requestBody = {
hash: "<byte>",
height: "<int64>",
}
ws.on('open', function() {
ws.send(JSON.stringify(requestBody));
});
ws.on('error', function(err) {
console.log('Error: ' + err);
});
ws.on('message', function(body) {
console.log(body);
});
// Console output (repeated for every message in the stream):
// {
// "hash": <byte>,
// "height": <int64>,
// }
```
+83 -15
View File
@@ -22,6 +22,7 @@ import (
"github.com/lightningnetwork/lnd/htlcswitch"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/keychain"
"github.com/lightningnetwork/lnd/labels"
"github.com/lightningnetwork/lnd/lnpeer"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/lnwallet"
@@ -66,6 +67,11 @@ const (
// in the real world.
MaxBtcFundingAmount = btcutil.Amount(1<<24) - 1
// MaxBtcFundingAmountWumbo is a soft-limit on the maximum size of wumbo
// channels. This limit is 10 BTC and is the only thing standing between
// you and limitless channel size (apart from 21 million cap)
MaxBtcFundingAmountWumbo = btcutil.Amount(1000000000)
// maxLtcFundingAmount is a soft-limit of the maximum channel size
// currently accepted on the Litecoin chain within the Lightning
// Protocol.
@@ -123,6 +129,7 @@ type reservationWithCtx struct {
remoteCsvDelay uint16
remoteMinHtlc lnwire.MilliSatoshi
remoteMaxValue lnwire.MilliSatoshi
remoteMaxHtlcs uint16
updateMtx sync.RWMutex
lastUpdated time.Time
@@ -243,6 +250,10 @@ type fundingConfig struct {
// transaction to the network.
PublishTransaction func(*wire.MsgTx, string) error
// UpdateLabel updates the label that a transaction has in our wallet,
// overwriting any existing labels.
UpdateLabel func(chainhash.Hash, string) error
// FeeEstimator calculates appropriate fee rates based on historical
// transaction information.
FeeEstimator chainfee.Estimator
@@ -354,6 +365,11 @@ type fundingConfig struct {
// due to fees.
MinChanSize btcutil.Amount
// MaxChanSize is the largest channel size that we'll accept as an
// inbound channel. We have such a parameter, so that you may decide how
// WUMBO you would like your channel.
MaxChanSize btcutil.Amount
// MaxPendingChannels is the maximum number of pending channels we
// allow for each peer.
MaxPendingChannels int
@@ -576,8 +592,15 @@ func (f *fundingManager) start() error {
channel.FundingOutpoint,
fundingTxBuf.Bytes())
// Set a nil short channel ID at this stage
// because we do not know it until our funding
// tx confirms.
label := labels.MakeLabel(
labels.LabelTypeChannelOpen, nil,
)
err = f.cfg.PublishTransaction(
channel.FundingTxn, "",
channel.FundingTxn, label,
)
if err != nil {
fndgLog.Errorf("Unable to rebroadcast "+
@@ -1193,13 +1216,24 @@ func (f *fundingManager) handleFundingOpen(fmsg *fundingOpenMsg) {
msg := fmsg.msg
amt := msg.FundingAmount
// We count the number of pending channels for this peer. This is the
// sum of the active reservations and the channels pending open in the
// database.
// We get all pending channels for this peer. This is the list of the
// active reservations and the channels pending open in the database.
f.resMtx.RLock()
numPending := len(f.activeReservations[peerIDKey])
reservations := f.activeReservations[peerIDKey]
// We don't count reservations that were created from a canned funding
// shim. The user has registered the shim and therefore expects this
// channel to arrive.
numPending := 0
for _, res := range reservations {
if !res.reservation.IsCannedShim() {
numPending++
}
}
f.resMtx.RUnlock()
// Also count the channels that are already pending. There we don't know
// the underlying intent anymore, unfortunately.
channels, err := f.cfg.Wallet.Cfg.Database.FetchOpenChannels(peerPubKey)
if err != nil {
f.failFundingFlow(
@@ -1209,7 +1243,13 @@ func (f *fundingManager) handleFundingOpen(fmsg *fundingOpenMsg) {
}
for _, c := range channels {
if c.IsPending {
// Pending channels that have a non-zero thaw height were also
// created through a canned funding shim. Those also don't
// count towards the DoS protection limit.
//
// TODO(guggero): Properly store the funding type (wallet, shim,
// PSBT) on the channel so we don't need to use the thaw height.
if c.IsPending && c.ThawHeight == 0 {
numPending++
}
}
@@ -1239,13 +1279,11 @@ func (f *fundingManager) handleFundingOpen(fmsg *fundingOpenMsg) {
return
}
// We'll reject any request to create a channel that's above the
// current soft-limit for channel size, but only if we're rejecting all
// wumbo channel initiations.
if f.cfg.NoWumboChans && msg.FundingAmount > MaxFundingAmount {
// Ensure that the remote party respects our maximum channel size.
if amt > f.cfg.MaxChanSize {
f.failFundingFlow(
fmsg.peer, fmsg.msg.PendingChannelID,
lnwire.ErrChanTooLarge,
lnwallet.ErrChanTooLarge(amt, f.cfg.MaxChanSize),
)
return
}
@@ -1399,6 +1437,7 @@ func (f *fundingManager) handleFundingOpen(fmsg *fundingOpenMsg) {
remoteCsvDelay: remoteCsvDelay,
remoteMinHtlc: minHtlc,
remoteMaxValue: remoteMaxValue,
remoteMaxHtlcs: maxHtlcs,
err: make(chan error, 1),
peer: fmsg.peer,
}
@@ -1548,7 +1587,6 @@ func (f *fundingManager) handleFundingAccept(fmsg *fundingAcceptMsg) {
// here so we can properly commit their accepted constraints to the
// reservation.
chanReserve := f.cfg.RequiredRemoteChanReserve(resCtx.chanAmt, msg.DustLimit)
maxHtlcs := f.cfg.RequiredRemoteMaxHTLCs(resCtx.chanAmt)
// The remote node has responded with their portion of the channel
// contribution. At this point, we can process their contribution which
@@ -1562,7 +1600,7 @@ func (f *fundingManager) handleFundingAccept(fmsg *fundingAcceptMsg) {
MaxPendingAmount: resCtx.remoteMaxValue,
ChanReserve: chanReserve,
MinHTLC: resCtx.remoteMinHtlc,
MaxAcceptedHtlcs: maxHtlcs,
MaxAcceptedHtlcs: resCtx.remoteMaxHtlcs,
CsvDelay: resCtx.remoteCsvDelay,
},
MultiSigKey: keychain.KeyDescriptor{
@@ -2032,7 +2070,13 @@ func (f *fundingManager) handleFundingSigned(fmsg *fundingSignedMsg) {
fndgLog.Infof("Broadcasting funding tx for ChannelPoint(%v): %x",
completeChan.FundingOutpoint, fundingTxBuf.Bytes())
err = f.cfg.PublishTransaction(fundingTx, "")
// Set a nil short channel ID at this stage because we do not
// know it until our funding tx confirms.
label := labels.MakeLabel(
labels.LabelTypeChannelOpen, nil,
)
err = f.cfg.PublishTransaction(fundingTx, label)
if err != nil {
fndgLog.Errorf("Unable to broadcast funding tx %x for "+
"ChannelPoint(%v): %v", fundingTxBuf.Bytes(),
@@ -2372,6 +2416,25 @@ func (f *fundingManager) handleFundingConfirmation(
fndgLog.Errorf("unable to report short chan id: %v", err)
}
// If we opened the channel, and lnd's wallet published our funding tx
// (which is not the case for some channels) then we update our
// transaction label with our short channel ID, which is known now that
// our funding transaction has confirmed. We do not label transactions
// we did not publish, because our wallet has no knowledge of them.
if completeChan.IsInitiator && completeChan.ChanType.HasFundingTx() {
shortChanID := completeChan.ShortChanID()
label := labels.MakeLabel(
labels.LabelTypeChannelOpen, &shortChanID,
)
err = f.cfg.UpdateLabel(
completeChan.FundingOutpoint.Hash, label,
)
if err != nil {
fndgLog.Errorf("unable to update label: %v", err)
}
}
// Close the discoverySignal channel, indicating to a separate
// goroutine that the channel now is marked as open in the database
// and that it is acceptable to process funding locked messages
@@ -3073,6 +3136,7 @@ func (f *fundingManager) handleInitFundingMsg(msg *initFundingMsg) {
minHtlcIn = msg.minHtlcIn
remoteCsvDelay = msg.remoteCsvDelay
maxValue = msg.maxValueInFlight
maxHtlcs = msg.maxHtlcs
)
// We'll determine our dust limit depending on which chain is active.
@@ -3211,6 +3275,10 @@ func (f *fundingManager) handleInitFundingMsg(msg *initFundingMsg) {
maxValue = f.cfg.RequiredRemoteMaxValue(capacity)
}
if maxHtlcs == 0 {
maxHtlcs = f.cfg.RequiredRemoteMaxHTLCs(capacity)
}
// If a pending channel map for this peer isn't already created, then
// we create one, ultimately allowing us to track this pending
// reservation within the target peer.
@@ -3225,6 +3293,7 @@ func (f *fundingManager) handleInitFundingMsg(msg *initFundingMsg) {
remoteCsvDelay: remoteCsvDelay,
remoteMinHtlc: minHtlcIn,
remoteMaxValue: maxValue,
remoteMaxHtlcs: maxHtlcs,
reservation: reservation,
peer: msg.peer,
updates: msg.updates,
@@ -3244,7 +3313,6 @@ func (f *fundingManager) handleInitFundingMsg(msg *initFundingMsg) {
// policy to determine of required commitment constraints for the
// remote party.
chanReserve := f.cfg.RequiredRemoteChanReserve(capacity, ourDustLimit)
maxHtlcs := f.cfg.RequiredRemoteMaxHTLCs(capacity)
fndgLog.Infof("Starting funding workflow with %v for pending_id(%x), "+
"committype=%v", msg.peer.Address(), chanID, commitType)
+104 -21
View File
@@ -33,6 +33,7 @@ import (
"github.com/lightningnetwork/lnd/keychain"
"github.com/lightningnetwork/lnd/lnpeer"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/lntest/mock"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
"github.com/lightningnetwork/lnd/lnwire"
@@ -54,6 +55,9 @@ const (
// maxPending is the maximum number of channels we allow opening to the
// same peer in the max pending channels test.
maxPending = 4
// A dummy value to use for the funding broadcast height.
fundingBroadcastHeight = 123
)
var (
@@ -99,6 +103,8 @@ var (
}
_, _ = testSig.R.SetString("63724406601629180062774974542967536251589935445068131219452686511677818569431", 10)
_, _ = testSig.S.SetString("18801056069249825825291287104931333862866033135609736119018462340006816851118", 10)
fundingNetParams = bitcoinTestNetParams
)
type mockNotifier struct {
@@ -282,7 +288,7 @@ func createTestFundingManager(t *testing.T, privKey *btcec.PrivateKey,
addr *lnwire.NetAddress, tempTestDir string,
options ...cfgOption) (*testNode, error) {
netParams := activeNetParams.Params
netParams := fundingNetParams.Params
estimator := chainfee.NewStaticEstimator(62500, 0)
chainNotifier := &mockNotifier{
@@ -296,14 +302,14 @@ func createTestFundingManager(t *testing.T, privKey *btcec.PrivateKey,
publTxChan := make(chan *wire.MsgTx, 1)
shutdownChan := make(chan struct{})
wc := &mockWalletController{
rootKey: alicePrivKey,
wc := &mock.WalletController{
RootKey: alicePrivKey,
}
signer := &mockSigner{
key: alicePrivKey,
signer := &mock.SingleSigner{
Privkey: alicePrivKey,
}
bio := &mockChainIO{
bestHeight: fundingBroadcastHeight,
bio := &mock.ChainIO{
BestHeight: fundingBroadcastHeight,
}
// The mock channel event notifier will receive events for each pending
@@ -323,8 +329,8 @@ func createTestFundingManager(t *testing.T, privKey *btcec.PrivateKey,
return nil, err
}
keyRing := &mockSecretKeyRing{
rootKey: alicePrivKey,
keyRing := &mock.SecretKeyRing{
RootKey: alicePrivKey,
}
lnw, err := createTestWallet(
@@ -421,8 +427,12 @@ func createTestFundingManager(t *testing.T, privKey *btcec.PrivateKey,
publTxChan <- txn
return nil
},
UpdateLabel: func(chainhash.Hash, string) error {
return nil
},
ZombieSweeperInterval: 1 * time.Hour,
ReservationTimeout: 1 * time.Nanosecond,
MaxChanSize: MaxFundingAmount,
MaxPendingChannels: lncfg.DefaultMaxPendingChannels,
NotifyOpenChannelEvent: evt.NotifyOpenChannelEvent,
OpenChannelPredicate: chainedAcceptor,
@@ -524,6 +534,9 @@ func recreateAliceFundingManager(t *testing.T, alice *testNode) {
publishChan <- txn
return nil
},
UpdateLabel: func(chainhash.Hash, string) error {
return nil
},
ZombieSweeperInterval: oldCfg.ZombieSweeperInterval,
ReservationTimeout: oldCfg.ReservationTimeout,
OpenChannelPredicate: chainedAcceptor,
@@ -643,7 +656,7 @@ func fundChannel(t *testing.T, alice, bob *testNode, localFundingAmt,
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
subtractFees: subtractFees,
localFundingAmt: localFundingAmt,
pushAmt: lnwire.NewMSatFromSatoshis(pushAmt),
@@ -1567,7 +1580,7 @@ func TestFundingManagerPeerTimeoutAfterInitFunding(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: 500000,
pushAmt: lnwire.NewMSatFromSatoshis(0),
private: false,
@@ -1629,7 +1642,7 @@ func TestFundingManagerPeerTimeoutAfterFundingOpen(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: 500000,
pushAmt: lnwire.NewMSatFromSatoshis(0),
private: false,
@@ -1700,7 +1713,7 @@ func TestFundingManagerPeerTimeoutAfterFundingAccept(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: 500000,
pushAmt: lnwire.NewMSatFromSatoshis(0),
private: false,
@@ -2424,7 +2437,7 @@ func TestFundingManagerCustomChannelParameters(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: localAmt,
pushAmt: lnwire.NewMSatFromSatoshis(pushAmt),
private: false,
@@ -2709,7 +2722,7 @@ func TestFundingManagerMaxPendingChannels(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: 5000000,
pushAmt: lnwire.NewMSatFromSatoshis(0),
private: false,
@@ -2879,7 +2892,7 @@ func TestFundingManagerRejectPush(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: 500000,
pushAmt: lnwire.NewMSatFromSatoshis(10),
private: true,
@@ -2936,7 +2949,7 @@ func TestFundingManagerMaxConfs(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: 500000,
pushAmt: lnwire.NewMSatFromSatoshis(10),
private: false,
@@ -3005,7 +3018,7 @@ func TestFundingManagerFundAll(t *testing.T) {
Value: btcutil.Amount(
0.05 * btcutil.SatoshiPerBitcoin,
),
PkScript: coinPkScript,
PkScript: mock.CoinPkScript,
OutPoint: wire.OutPoint{
Hash: chainhash.Hash{},
Index: 0,
@@ -3016,7 +3029,7 @@ func TestFundingManagerFundAll(t *testing.T) {
Value: btcutil.Amount(
0.06 * btcutil.SatoshiPerBitcoin,
),
PkScript: coinPkScript,
PkScript: mock.CoinPkScript,
OutPoint: wire.OutPoint{
Hash: chainhash.Hash{},
Index: 1,
@@ -3050,7 +3063,7 @@ func TestFundingManagerFundAll(t *testing.T) {
alice, bob := setupFundingManagers(t)
defer tearDownFundingManagers(t, alice, bob)
alice.fundingMgr.cfg.Wallet.WalletController.(*mockWalletController).utxos = allCoins
alice.fundingMgr.cfg.Wallet.WalletController.(*mock.WalletController).Utxos = allCoins
// We will consume the channel updates as we go, so no
// buffering is needed.
@@ -3200,6 +3213,75 @@ func expectOpenChannelMsg(t *testing.T, msgChan chan lnwire.Message) *lnwire.Ope
return openChannelReq
}
func TestMaxChannelSizeConfig(t *testing.T) {
t.Parallel()
// Create a set of funding managers that will reject wumbo
// channels but set --maxchansize explicitly lower than soft-limit.
// Verify that wumbo rejecting funding managers will respect --maxchansize
// below 16777215 satoshi (MaxFundingAmount) limit.
alice, bob := setupFundingManagers(t, func(cfg *fundingConfig) {
cfg.NoWumboChans = true
cfg.MaxChanSize = MaxFundingAmount - 1
})
// Attempt to create a channel above the limit
// imposed by --maxchansize, which should be rejected.
updateChan := make(chan *lnrpc.OpenStatusUpdate)
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: MaxFundingAmount,
pushAmt: lnwire.NewMSatFromSatoshis(0),
private: false,
updates: updateChan,
err: errChan,
}
// After processing the funding open message, bob should respond with
// an error rejecting the channel that exceeds size limit.
alice.fundingMgr.initFundingWorkflow(bob, initReq)
openChanMsg := expectOpenChannelMsg(t, alice.msgChan)
bob.fundingMgr.processFundingOpen(openChanMsg, alice)
assertErrorSent(t, bob.msgChan)
// Create a set of funding managers that will reject wumbo
// channels but set --maxchansize explicitly higher than soft-limit
// A --maxchansize greater than this limit should have no effect.
tearDownFundingManagers(t, alice, bob)
alice, bob = setupFundingManagers(t, func(cfg *fundingConfig) {
cfg.NoWumboChans = true
cfg.MaxChanSize = MaxFundingAmount + 1
})
// We expect Bob to respond with an Accept channel message.
alice.fundingMgr.initFundingWorkflow(bob, initReq)
openChanMsg = expectOpenChannelMsg(t, alice.msgChan)
bob.fundingMgr.processFundingOpen(openChanMsg, alice)
assertFundingMsgSent(t, bob.msgChan, "AcceptChannel")
// Verify that wumbo accepting funding managers will respect --maxchansize
// Create the funding managers, this time allowing
// wumbo channels but setting --maxchansize explicitly.
tearDownFundingManagers(t, alice, bob)
alice, bob = setupFundingManagers(t, func(cfg *fundingConfig) {
cfg.NoWumboChans = false
cfg.MaxChanSize = btcutil.Amount(100000000)
})
// Attempt to create a channel above the limit
// imposed by --maxchansize, which should be rejected.
initReq.localFundingAmt = btcutil.SatoshiPerBitcoin + 1
// After processing the funding open message, bob should respond with
// an error rejecting the channel that exceeds size limit.
alice.fundingMgr.initFundingWorkflow(bob, initReq)
openChanMsg = expectOpenChannelMsg(t, alice.msgChan)
bob.fundingMgr.processFundingOpen(openChanMsg, alice)
assertErrorSent(t, bob.msgChan)
}
// TestWumboChannelConfig tests that the funding manager will respect the wumbo
// channel config param when creating or accepting new channels.
func TestWumboChannelConfig(t *testing.T) {
@@ -3218,7 +3300,7 @@ func TestWumboChannelConfig(t *testing.T) {
errChan := make(chan error, 1)
initReq := &openChanReq{
targetPubkey: bob.privKey.PubKey(),
chainHash: *activeNetParams.GenesisHash,
chainHash: *fundingNetParams.GenesisHash,
localFundingAmt: MaxFundingAmount,
pushAmt: lnwire.NewMSatFromSatoshis(0),
private: false,
@@ -3248,6 +3330,7 @@ func TestWumboChannelConfig(t *testing.T) {
tearDownFundingManagers(t, alice, bob)
alice, bob = setupFundingManagers(t, func(cfg *fundingConfig) {
cfg.NoWumboChans = false
cfg.MaxChanSize = MaxBtcFundingAmountWumbo
})
// We should now be able to initiate a wumbo channel funding w/o any
+6 -5
View File
@@ -5,11 +5,11 @@ require (
github.com/NebulousLabs/fastrand v0.0.0-20181203155948-6fb6489aac4e // indirect
github.com/NebulousLabs/go-upnp v0.0.0-20180202185039-29b680b06c82
github.com/Yawning/aez v0.0.0-20180114000226-4dad034d9db2
github.com/btcsuite/btcd v0.20.1-beta.0.20200515232429-9f0179fd2c46
github.com/btcsuite/btcd v0.20.1-beta.0.20200730232343-1db1b6f8217f
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
github.com/btcsuite/btcutil v1.0.2
github.com/btcsuite/btcutil/psbt v1.0.2
github.com/btcsuite/btcwallet v0.11.1-0.20200612012534-48addcd5591a
github.com/btcsuite/btcutil/psbt v1.0.3-0.20200826194809-5f93e33af2b0
github.com/btcsuite/btcwallet v0.11.1-0.20200904022754-2c5947a45222
github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0
github.com/btcsuite/btcwallet/wallet/txrules v1.0.0
github.com/btcsuite/btcwallet/walletdb v1.3.3
@@ -48,7 +48,7 @@ require (
github.com/lightninglabs/neutrino v0.11.1-0.20200316235139-bffc52e8f200
github.com/lightninglabs/protobuf-hex-display v1.3.3-0.20191212020323-b444784ce75d
github.com/lightningnetwork/lightning-onion v1.0.2-0.20200501022730-3c8c8d0b89ea
github.com/lightningnetwork/lnd/cert v1.0.2
github.com/lightningnetwork/lnd/cert v1.0.3
github.com/lightningnetwork/lnd/clock v1.0.1
github.com/lightningnetwork/lnd/queue v1.0.4
github.com/lightningnetwork/lnd/ticker v1.0.0
@@ -66,6 +66,7 @@ require (
go.uber.org/zap v1.14.1 // indirect
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2
google.golang.org/grpc v1.24.0
gopkg.in/errgo.v1 v1.0.1 // indirect
@@ -85,4 +86,4 @@ replace github.com/lightningnetwork/lnd/clock => ./clock
replace git.schwanenlied.me/yawning/bsaes.git => github.com/Yawning/bsaes v0.0.0-20180720073208-c0276d75487e
go 1.12
go 1.13
+9 -8
View File
@@ -27,18 +27,18 @@ github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcug
github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.20.1-beta.0.20200513120220-b470eee47728/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.20.1-beta.0.20200515232429-9f0179fd2c46 h1:QyTpiR5nQe94vza2qkvf7Ns8XX2Rjh/vdIhO3RzGj4o=
github.com/btcsuite/btcd v0.20.1-beta.0.20200515232429-9f0179fd2c46/go.mod h1:Yktc19YNjh/Iz2//CX0vfRTS4IJKM/RKO5YZ9Fn+Pgo=
github.com/btcsuite/btcd v0.20.1-beta.0.20200730232343-1db1b6f8217f h1:m/GhMTvDQLbID616c4TYdHyt0MZ9lH5B/nf9Lu3okCY=
github.com/btcsuite/btcd v0.20.1-beta.0.20200730232343-1db1b6f8217f/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts=
github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts=
github.com/btcsuite/btcutil/psbt v1.0.2 h1:gCVY3KxdoEVU7Q6TjusPO+GANIwVgr9yTLqM+a6CZr8=
github.com/btcsuite/btcutil/psbt v1.0.2/go.mod h1:LVveMu4VaNSkIRTZu2+ut0HDBRuYjqGocxDMNS1KuGQ=
github.com/btcsuite/btcwallet v0.11.1-0.20200612012534-48addcd5591a h1:AZ1Mf0gd9mgJqrTTIFUc17ep9EKUbQusVAIzJ6X+x3Q=
github.com/btcsuite/btcwallet v0.11.1-0.20200612012534-48addcd5591a/go.mod h1:9+AH3V5mcTtNXTKe+fe63fDLKGOwQbZqmvOVUef+JFE=
github.com/btcsuite/btcutil/psbt v1.0.3-0.20200826194809-5f93e33af2b0 h1:3Zumkyl6PWyHuVJ04me0xeD9CnPOhNgeGpapFbzy7O4=
github.com/btcsuite/btcutil/psbt v1.0.3-0.20200826194809-5f93e33af2b0/go.mod h1:LVveMu4VaNSkIRTZu2+ut0HDBRuYjqGocxDMNS1KuGQ=
github.com/btcsuite/btcwallet v0.11.1-0.20200904022754-2c5947a45222 h1:rh1FQAhh+BeR29twIFDM0RLOFpDK62tsABtUkWctTXw=
github.com/btcsuite/btcwallet v0.11.1-0.20200904022754-2c5947a45222/go.mod h1:owv9oZqM0HnUW+ByF7VqOgfs2eb0ooiePW/+Tl/i/Nk=
github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 h1:KGHMW5sd7yDdDMkCZ/JpP0KltolFsQcB973brBnfj4c=
github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU=
github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 h1:2VsfS0sBedcM5KmDzRMT3+b6xobqWveZGvjb+jFez5w=
@@ -47,8 +47,6 @@ github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0 h1:6DxkcoMnCPY4E9cUDPB5tbuuf
github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs=
github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
github.com/btcsuite/btcwallet/walletdb v1.2.0/go.mod h1:9cwc1Yyg4uvd4ZdfdoMnALji+V9gfWSMfxEdLdR5Vwc=
github.com/btcsuite/btcwallet/walletdb v1.3.1 h1:lW1Ac3F1jJY4K11P+YQtRNcP5jFk27ASfrV7C6mvRU0=
github.com/btcsuite/btcwallet/walletdb v1.3.1/go.mod h1:9cwc1Yyg4uvd4ZdfdoMnALji+V9gfWSMfxEdLdR5Vwc=
github.com/btcsuite/btcwallet/walletdb v1.3.2 h1:nFnMBVgkqoVOx08Z756oDwNc9sdVgYR52T1ONSXs90w=
github.com/btcsuite/btcwallet/walletdb v1.3.2/go.mod h1:GZCMPNpUu5KE3ASoVd+k06p/1OW8OwNGCCaNWRto2cQ=
github.com/btcsuite/btcwallet/walletdb v1.3.3 h1:u6e7vRIKBF++cJy+hOHaMGg+88ZTwvpaY27AFvtB668=
@@ -86,6 +84,8 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/decred/dcrd/lru v1.0.0 h1:Kbsb1SFDsIlaupWPwsPp+dkxiBY1frcS07PCPgotKz8=
github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -332,6 +332,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFW
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c h1:UIcGWL6/wpCfyGuJnRFJRurA+yj8RrW7Q6x2YMCXt6c=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+18
View File
@@ -0,0 +1,18 @@
// +build !windows,!solaris,!netbsd,!openbsd
package healthcheck
import "syscall"
// AvailableDiskSpace returns ratio of available disk space to total capacity.
func AvailableDiskSpace(path string) (float64, error) {
s := syscall.Statfs_t{}
err := syscall.Statfs(path, &s)
if err != nil {
return 0, err
}
// Calculate our free blocks/total blocks to get our total ratio of
// free blocks.
return float64(s.Bfree) / float64(s.Blocks), nil
}
Loaded 100 of 258 files, more files were not shown because too many files have changed in this diff. Show more