misc: add ability to store the wallet in the remote DB

This commit is contained in:
Andras Banki-Horvath 2021-03-04 23:13:06 +01:00
parent 08be03367a
commit b6a620e6b2
No known key found for this signature in database
GPG Key ID: 80E5375C094198D8
7 changed files with 203 additions and 57 deletions

@ -106,14 +106,13 @@ type Config struct {
// optional.
FeeURL string
// DBTimeOut specifies the timeout value to use when opening the wallet
// database.
DBTimeOut time.Duration
// Dialer is a function closure that will be used to establish outbound
// TCP connections to Bitcoin peers in the event of a pruned block being
// requested.
Dialer chain.Dialer
// LoaderOptions holds functional wallet db loader options.
LoaderOptions []btcwallet.LoaderOption
}
const (
@ -283,11 +282,10 @@ func NewChainControl(cfg *Config, blockCache *blockcache.BlockCache) (
PublicPass: cfg.PublicWalletPw,
Birthday: cfg.Birthday,
RecoveryWindow: cfg.RecoveryWindow,
DataDir: homeChainConfig.ChainDir,
NetParams: cfg.ActiveNetParams.Params,
CoinType: cfg.ActiveNetParams.CoinType,
Wallet: cfg.Wallet,
DBTimeOut: cfg.DBTimeOut,
LoaderOptions: cfg.LoaderOptions,
}
var err error

67
lnd.go

@ -375,9 +375,38 @@ func Main(cfg *Config, lisCfg ListenerCfg, interceptor signal.Interceptor) error
defer cleanUp()
var loaderOpt btcwallet.LoaderOption
if cfg.Cluster.EnableLeaderElection {
// The wallet loader will attempt to use/create the wallet in
// the replicated remote DB if we're running in a clustered
// environment. This will ensure that all members of the cluster
// have access to the same wallet state.
loaderOpt = btcwallet.LoaderWithExternalWalletDB(
remoteChanDB.Backend,
)
} else {
// When "running locally", LND will use the bbolt wallet.db to
// store the wallet located in the chain data dir, parametrized
// by the active network.
chainConfig := cfg.Bitcoin
if cfg.registeredChains.PrimaryChain() == chainreg.LitecoinChain {
chainConfig = cfg.Litecoin
}
dbDirPath := btcwallet.NetworkDir(
chainConfig.ChainDir, cfg.ActiveNetParams.Params,
)
loaderOpt = btcwallet.LoaderWithLocalWalletDB(
dbDirPath, !cfg.SyncFreelist, cfg.DB.Bolt.DBTimeout,
)
}
// We'll create the WalletUnlockerService and check whether the wallet
// already exists.
pwService := createWalletUnlockerService(cfg)
pwService := createWalletUnlockerService(cfg,
[]btcwallet.LoaderOption{loaderOpt},
)
walletExists, err := pwService.WalletExists()
if err != nil {
return err
@ -448,7 +477,10 @@ func Main(cfg *Config, lisCfg ListenerCfg, interceptor signal.Interceptor) error
// started with the --noseedbackup flag, we use the default password
// for wallet encryption.
if !cfg.NoSeedBackup {
params, err := waitForWalletPassword(cfg, pwService, interceptor.ShutdownChannel())
params, err := waitForWalletPassword(
cfg, pwService, []btcwallet.LoaderOption{loaderOpt},
interceptor.ShutdownChannel(),
)
if err != nil {
err := fmt.Errorf("unable to set up wallet password "+
"listeners: %v", err)
@ -598,7 +630,6 @@ func Main(cfg *Config, lisCfg ListenerCfg, interceptor signal.Interceptor) error
Birthday: walletInitParams.Birthday,
RecoveryWindow: walletInitParams.RecoveryWindow,
Wallet: walletInitParams.Wallet,
DBTimeOut: cfg.DB.Bolt.DBTimeout,
NeutrinoCS: neutrinoCS,
ActiveNetParams: cfg.ActiveNetParams,
FeeURL: cfg.FeeURL,
@ -606,6 +637,9 @@ func Main(cfg *Config, lisCfg ListenerCfg, interceptor signal.Interceptor) error
return cfg.net.Dial("tcp", addr, cfg.ConnectionTimeout)
},
BlockCacheSize: cfg.BlockCacheSize,
LoaderOptions: []btcwallet.LoaderOption{
loaderOpt,
},
}
activeChainControl, cleanup, err := chainreg.NewChainControl(
@ -1200,7 +1234,9 @@ type WalletUnlockParams struct {
// createWalletUnlockerService creates a WalletUnlockerService from the passed
// config.
func createWalletUnlockerService(cfg *Config) *walletunlocker.UnlockerService {
func createWalletUnlockerService(cfg *Config,
loaderOpts []btcwallet.LoaderOption) *walletunlocker.UnlockerService {
chainConfig := cfg.Bitcoin
if cfg.registeredChains.PrimaryChain() == chainreg.LitecoinChain {
chainConfig = cfg.Litecoin
@ -1212,10 +1248,11 @@ func createWalletUnlockerService(cfg *Config) *walletunlocker.UnlockerService {
macaroonFiles := []string{
cfg.AdminMacPath, cfg.ReadMacPath, cfg.InvoiceMacPath,
}
return walletunlocker.New(
chainConfig.ChainDir, cfg.ActiveNetParams.Params,
!cfg.SyncFreelist, macaroonFiles, cfg.DB.Bolt.DBTimeout,
cfg.ResetWalletTransactions,
cfg.ResetWalletTransactions, loaderOpts,
)
}
@ -1382,12 +1419,8 @@ func startRestProxy(cfg *Config, rpcServer *rpcServer, restDialOpts []grpc.DialO
// this RPC server.
func waitForWalletPassword(cfg *Config,
pwService *walletunlocker.UnlockerService,
shutdownChan <-chan struct{}) (*WalletUnlockParams, error) {
chainConfig := cfg.Bitcoin
if cfg.registeredChains.PrimaryChain() == chainreg.LitecoinChain {
chainConfig = cfg.Litecoin
}
loaderOpts []btcwallet.LoaderOption, shutdownChan <-chan struct{}) (
*WalletUnlockParams, error) {
// Wait for user to provide the password.
ltndLog.Infof("Waiting for wallet encryption password. Use `lncli " +
@ -1419,13 +1452,13 @@ func waitForWalletPassword(cfg *Config,
keychain.KeyDerivationVersion)
}
netDir := btcwallet.NetworkDir(
chainConfig.ChainDir, cfg.ActiveNetParams.Params,
)
loader := wallet.NewLoader(
cfg.ActiveNetParams.Params, netDir, !cfg.SyncFreelist,
cfg.DB.Bolt.DBTimeout, recoveryWindow,
loader, err := btcwallet.NewWalletLoader(
cfg.ActiveNetParams.Params, recoveryWindow,
loaderOpts...,
)
if err != nil {
return nil, err
}
// With the seed, we can now use the wallet loader to create
// the wallet, then pass it back to avoid unlocking it again.

@ -19,12 +19,14 @@ import (
"github.com/btcsuite/btcutil/psbt"
"github.com/btcsuite/btcwallet/chain"
"github.com/btcsuite/btcwallet/waddrmgr"
"github.com/btcsuite/btcwallet/wallet"
base "github.com/btcsuite/btcwallet/wallet"
"github.com/btcsuite/btcwallet/wallet/txauthor"
"github.com/btcsuite/btcwallet/wallet/txrules"
"github.com/btcsuite/btcwallet/walletdb"
"github.com/btcsuite/btcwallet/wtxmgr"
"github.com/lightningnetwork/lnd/blockcache"
"github.com/lightningnetwork/lnd/channeldb/kvdb"
"github.com/lightningnetwork/lnd/keychain"
"github.com/lightningnetwork/lnd/lnwallet"
"github.com/lightningnetwork/lnd/lnwallet/chainfee"
@ -37,6 +39,13 @@ const (
// UnconfirmedHeight is the special case end height that is used to
// obtain unconfirmed transactions from ListTransactionDetails.
UnconfirmedHeight int32 = -1
// walletMetaBucket is used to store wallet metadata.
walletMetaBucket = "lnwallet"
// walletReadyKey is used to indicate that the wallet has been
// initialized.
walletReadyKey = "ready"
)
var (
@ -87,9 +96,6 @@ var _ lnwallet.BlockChainIO = (*BtcWallet)(nil)
// New returns a new fully initialized instance of BtcWallet given a valid
// configuration struct.
func New(cfg Config, blockCache *blockcache.BlockCache) (*BtcWallet, error) {
// Ensure the wallet exists or create it when the create flag is set.
netDir := NetworkDir(cfg.DataDir, cfg.NetParams)
// Create the key scope for the coin type being managed by this wallet.
chainKeyScope := waddrmgr.KeyScope{
Purpose: keychain.BIP0043Purpose,
@ -108,10 +114,13 @@ func New(cfg Config, blockCache *blockcache.BlockCache) (*BtcWallet, error) {
} else {
pubPass = cfg.PublicPass
}
loader := base.NewLoader(
cfg.NetParams, netDir, cfg.NoFreelistSync,
cfg.DBTimeOut, cfg.RecoveryWindow,
loader, err := NewWalletLoader(
cfg.NetParams, cfg.RecoveryWindow, cfg.LoaderOptions...,
)
if err != nil {
return nil, err
}
walletExists, err := loader.WalletExists()
if err != nil {
return nil, err
@ -149,6 +158,104 @@ func New(cfg Config, blockCache *blockcache.BlockCache) (*BtcWallet, error) {
}, nil
}
// loaderCfg holds optional wallet loader configuration.
type loaderCfg struct {
dbDirPath string
noFreelistSync bool
dbTimeout time.Duration
useLocalDB bool
externalDB kvdb.Backend
}
// LoaderOption is a functional option to update the optional loader config.
type LoaderOption func(*loaderCfg)
// LoaderWithLocalWalletDB configures the wallet loader to use the local db.
func LoaderWithLocalWalletDB(dbDirPath string, noFreelistSync bool,
dbTimeout time.Duration) LoaderOption {
return func(cfg *loaderCfg) {
cfg.dbDirPath = dbDirPath
cfg.noFreelistSync = noFreelistSync
cfg.dbTimeout = dbTimeout
cfg.useLocalDB = true
}
}
// LoaderWithExternalWalletDB configures the wallet loadr to use an external db.
func LoaderWithExternalWalletDB(db kvdb.Backend) LoaderOption {
return func(cfg *loaderCfg) {
cfg.externalDB = db
}
}
// NewWalletLoader constructs a wallet loader.
func NewWalletLoader(chainParams *chaincfg.Params, recoveryWindow uint32,
opts ...LoaderOption) (*wallet.Loader, error) {
cfg := &loaderCfg{}
// Apply all functional options.
for _, o := range opts {
o(cfg)
}
if cfg.externalDB != nil && cfg.useLocalDB {
return nil, fmt.Errorf("wallet can either be in the local or " +
"an external db")
}
if cfg.externalDB != nil {
loader, err := base.NewLoaderWithDB(
chainParams, recoveryWindow, cfg.externalDB,
func() (bool, error) {
return externalWalletExists(cfg.externalDB)
},
)
if err != nil {
return nil, err
}
// Decorate wallet db with out own key such that we
// can always check whether the wallet exists or not.
loader.OnWalletCreated(onWalletCreated)
return loader, nil
}
return base.NewLoader(
chainParams, cfg.dbDirPath, cfg.noFreelistSync,
cfg.dbTimeout, recoveryWindow,
), nil
}
// externalWalletExists is a helper function that we use to template btcwallet's
// Loader in order to be able check if the wallet database has been initialized
// in an external DB.
func externalWalletExists(db kvdb.Backend) (bool, error) {
exists := false
err := kvdb.View(db, func(tx kvdb.RTx) error {
metaBucket := tx.ReadBucket([]byte(walletMetaBucket))
if metaBucket != nil {
walletReady := metaBucket.Get([]byte(walletReadyKey))
exists = string(walletReady) == walletReadyKey
}
return nil
}, func() {})
return exists, err
}
// onWalletCreated is executed when btcwallet creates the wallet the first time.
func onWalletCreated(tx kvdb.RwTx) error {
metaBucket, err := tx.CreateTopLevelBucket([]byte(walletMetaBucket))
if err != nil {
return err
}
return metaBucket.Put([]byte(walletReadyKey), []byte(walletReadyKey))
}
// BackEnd returns the underlying ChainService's name as a string.
//
// This is a part of the WalletController interface.

@ -28,10 +28,6 @@ var (
// Config is a struct which houses configuration parameters which modify the
// instance of BtcWallet generated by the New() function.
type Config struct {
// DataDir is the name of the directory where the wallet's persistent
// state should be stored.
DataDir string
// LogDir is the name of the directory which should be used to store
// generated log files.
LogDir string
@ -76,14 +72,8 @@ type Config struct {
// normally when creating the BtcWallet.
Wallet *wallet.Wallet
// NoFreelistSync, if true, prevents the database from syncing its
// freelist to disk, resulting in improved performance at the expense of
// increased startup time.
NoFreelistSync bool
// DBTimeOut specifies the timeout value to use when opening the wallet
// database.
DBTimeOut time.Duration
// LoaderOptions holds functional wallet db loader options.
LoaderOptions []LoaderOption
}
// NetworkDir returns the directory name of a network directory to hold wallet

@ -3427,12 +3427,16 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
aliceWalletConfig := &btcwallet.Config{
PrivatePass: []byte("alice-pass"),
HdSeed: aliceSeedBytes,
DataDir: tempTestDirAlice,
NetParams: netParams,
ChainSource: aliceClient,
CoinType: keychain.CoinTypeTestnet,
// wallet starts in recovery mode
RecoveryWindow: 2,
LoaderOptions: []btcwallet.LoaderOption{
btcwallet.LoaderWithLocalWalletDB(
tempTestDirAlice, false, time.Minute,
),
},
}
aliceWalletController, err = walletDriver.New(
aliceWalletConfig, blockCache,
@ -3454,12 +3458,16 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
bobWalletConfig := &btcwallet.Config{
PrivatePass: []byte("bob-pass"),
HdSeed: bobSeedBytes,
DataDir: tempTestDirBob,
NetParams: netParams,
ChainSource: bobClient,
CoinType: keychain.CoinTypeTestnet,
// wallet starts without recovery mode
RecoveryWindow: 0,
LoaderOptions: []btcwallet.LoaderOption{
btcwallet.LoaderWithLocalWalletDB(
tempTestDirBob, false, time.Minute,
),
},
}
bobWalletController, err = walletDriver.New(
bobWalletConfig, blockCache,

@ -137,12 +137,16 @@ type UnlockerService struct {
// resetWalletTransactions indicates that the wallet state should be
// reset on unlock to force a full chain rescan.
resetWalletTransactions bool
// LoaderOpts holds the functional options for the wallet loader.
loaderOpts []btcwallet.LoaderOption
}
// New creates and returns a new UnlockerService.
func New(chainDir string, params *chaincfg.Params, noFreelistSync bool,
macaroonFiles []string, dbTimeout time.Duration,
resetWalletTransactions bool) *UnlockerService {
resetWalletTransactions bool,
loaderOpts []btcwallet.LoaderOption) *UnlockerService {
return &UnlockerService{
InitMsgs: make(chan *WalletInitMsg, 1),
@ -157,17 +161,16 @@ func New(chainDir string, params *chaincfg.Params, noFreelistSync bool,
dbTimeout: dbTimeout,
noFreelistSync: noFreelistSync,
resetWalletTransactions: resetWalletTransactions,
loaderOpts: loaderOpts,
}
}
func (u *UnlockerService) newLoader(recoveryWindow uint32) (*wallet.Loader,
error) {
netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
return wallet.NewLoader(
u.netParams, netDir, u.noFreelistSync, u.dbTimeout,
recoveryWindow,
), nil
return btcwallet.NewWalletLoader(
u.netParams, recoveryWindow, u.loaderOpts...,
)
}
// WalletExists returns whether a wallet exists on the file path the

@ -48,6 +48,13 @@ var (
)
)
func testLoaderOpts(testDir string) []btcwallet.LoaderOption {
dbDir := btcwallet.NetworkDir(testDir, testNetParams)
return []btcwallet.LoaderOption{
btcwallet.LoaderWithLocalWalletDB(dbDir, true, time.Minute),
}
}
func createTestWallet(t *testing.T, dir string, netParams *chaincfg.Params) {
createTestWalletWithPw(t, testPassword, testPassword, dir, netParams)
}
@ -148,7 +155,7 @@ func TestGenSeed(t *testing.T) {
service := walletunlocker.New(
testDir, testNetParams, true, nil, kvdb.DefaultDBTimeout,
false,
false, testLoaderOpts(testDir),
)
// Now that the service has been created, we'll ask it to generate a
@ -186,7 +193,7 @@ func TestGenSeedGenerateEntropy(t *testing.T) {
}()
service := walletunlocker.New(
testDir, testNetParams, true, nil, kvdb.DefaultDBTimeout,
false,
false, testLoaderOpts(testDir),
)
// Now that the service has been created, we'll ask it to generate a
@ -223,7 +230,7 @@ func TestGenSeedInvalidEntropy(t *testing.T) {
}()
service := walletunlocker.New(
testDir, testNetParams, true, nil, kvdb.DefaultDBTimeout,
false,
false, testLoaderOpts(testDir),
)
// Now that the service has been created, we'll ask it to generate a
@ -257,7 +264,7 @@ func TestInitWallet(t *testing.T) {
// Create new UnlockerService.
service := walletunlocker.New(
testDir, testNetParams, true, nil, kvdb.DefaultDBTimeout,
false,
false, testLoaderOpts(testDir),
)
// Once we have the unlocker service created, we'll now instantiate a
@ -346,7 +353,7 @@ func TestCreateWalletInvalidEntropy(t *testing.T) {
// Create new UnlockerService.
service := walletunlocker.New(
testDir, testNetParams, true, nil, kvdb.DefaultDBTimeout,
false,
false, testLoaderOpts(testDir),
)
// We'll attempt to init the wallet with an invalid cipher seed and
@ -379,7 +386,7 @@ func TestUnlockWallet(t *testing.T) {
// unlock.
service := walletunlocker.New(
testDir, testNetParams, true, nil, kvdb.DefaultDBTimeout,
true,
true, testLoaderOpts(testDir),
)
ctx := context.Background()
@ -471,7 +478,7 @@ func TestChangeWalletPasswordNewRootkey(t *testing.T) {
// Create a new UnlockerService with our temp files.
service := walletunlocker.New(
testDir, testNetParams, true, tempFiles, kvdb.DefaultDBTimeout,
false,
false, testLoaderOpts(testDir),
)
ctx := context.Background()
@ -583,7 +590,7 @@ func TestChangeWalletPasswordStateless(t *testing.T) {
service := walletunlocker.New(
testDir, testNetParams, true, []string{
tempMacFile, nonExistingFile,
}, kvdb.DefaultDBTimeout, false,
}, kvdb.DefaultDBTimeout, false, testLoaderOpts(testDir),
)
// Create a wallet we can try to unlock. We use the default password