diff --git a/chainntnfs/test_utils.go b/chainntnfs/test_utils.go index 715eae75..a9d95eaa 100644 --- a/chainntnfs/test_utils.go +++ b/chainntnfs/test_utils.go @@ -268,7 +268,7 @@ func NewNeutrinoBackend(t *testing.T, minerAddr string) (*neutrino.ChainService, } dbName := filepath.Join(spvDir, "neutrino.db") - spvDatabase, err := walletdb.Create("bdb", dbName) + spvDatabase, err := walletdb.Create("bdb", dbName, true) if err != nil { os.RemoveAll(spvDir) t.Fatalf("unable to create walletdb: %v", err) diff --git a/chainregistry.go b/chainregistry.go index a209cc66..7be92afd 100644 --- a/chainregistry.go +++ b/chainregistry.go @@ -703,7 +703,7 @@ func initNeutrinoBackend(chainDir string) (*neutrino.ChainService, func(), error } dbName := filepath.Join(dbPath, "neutrino.db") - db, err := walletdb.Create("bdb", dbName) + db, err := walletdb.Create("bdb", dbName, !cfg.SyncFreelist) if err != nil { return nil, nil, fmt.Errorf("unable to create neutrino "+ "database: %v", err) diff --git a/channeldb/db.go b/channeldb/db.go index 4bcfa0f8..667ea418 100644 --- a/channeldb/db.go +++ b/channeldb/db.go @@ -85,7 +85,7 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) { // Specify bbolt freelist options to reduce heap pressure in case the // freelist grows to be very large. options := &bbolt.Options{ - NoFreelistSync: true, + NoFreelistSync: opts.NoFreelistSync, FreelistType: bbolt.FreelistMapType, } diff --git a/channeldb/options.go b/channeldb/options.go index a96ebfa9..38ac05ef 100644 --- a/channeldb/options.go +++ b/channeldb/options.go @@ -21,6 +21,11 @@ type Options struct { // ChannelCacheSize is the maximum number of ChannelEdges to hold in the // channel cache. ChannelCacheSize int + + // NoFreelistSync, if true, prevents the database from syncing its + // freelist to disk, resulting in improved performance at the expense of + // increased startup time. + NoFreelistSync bool } // DefaultOptions returns an Options populated with default values. @@ -28,6 +33,7 @@ func DefaultOptions() Options { return Options{ RejectCacheSize: DefaultRejectCacheSize, ChannelCacheSize: DefaultChannelCacheSize, + NoFreelistSync: true, } } @@ -47,3 +53,10 @@ func OptionSetChannelCacheSize(n int) OptionModifier { o.ChannelCacheSize = n } } + +// OptionSetSyncFreelist allows the database to sync its freelist. +func OptionSetSyncFreelist(b bool) OptionModifier { + return func(o *Options) { + o.NoFreelistSync = !b + } +} diff --git a/config.go b/config.go index 0800345c..dc71a75c 100644 --- a/config.go +++ b/config.go @@ -234,6 +234,7 @@ type config struct { LndDir string `long:"lnddir" description:"The base directory that contains lnd's data, logs, configuration file, etc."` ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` DataDir string `short:"b" long:"datadir" description:"The directory to store lnd's data within"` + SyncFreelist bool `long:"sync-freelist" description:"Whether the databases used within lnd should sync their freelist to disk. This is disabled by default resulting in improved memory performance during operation, but with an increase in startup time."` TLSCertPath string `long:"tlscertpath" description:"Path to write the TLS certificate for lnd's RPC and REST services"` TLSKeyPath string `long:"tlskeypath" description:"Path to write the TLS private key for lnd's RPC and REST services"` TLSExtraIPs []string `long:"tlsextraip" description:"Adds an extra ip to the generated certificate"` diff --git a/go.mod b/go.mod index d6aea561..0a0188dc 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,10 @@ require ( github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d - github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953 + github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 - github.com/btcsuite/btcwallet/walletdb v1.0.0 + github.com/btcsuite/btcwallet/walletdb v1.1.0 github.com/btcsuite/btcwallet/wtxmgr v1.0.0 github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 github.com/coreos/bbolt v1.3.3 diff --git a/go.sum b/go.sum index e9ed3de6..9865e20b 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953 h1:NG3SmXd3KMOF4/BHVQaJuayrlXBosJgwUjeHcX4k198= -github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI= +github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb h1:W4l92i/ImG2qx4msNWSzhKqnvfQM3d8weQ27D++lhxk= +github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI= github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 h1:KGHMW5sd7yDdDMkCZ/JpP0KltolFsQcB973brBnfj4c= github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU= github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 h1:2VsfS0sBedcM5KmDzRMT3+b6xobqWveZGvjb+jFez5w= @@ -35,6 +35,8 @@ github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0 h1:6DxkcoMnCPY4E9cUDPB5tbuuf github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs= github.com/btcsuite/btcwallet/walletdb v1.0.0 h1:mheT7vCWK5EP6rZzhxsQ7ms9+yX4VE8bwiJctECBeNw= github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk= +github.com/btcsuite/btcwallet/walletdb v1.1.0 h1:JHAL7wZ8pX4SULabeAv/wPO9sseRWMGzE80lfVmRw6Y= +github.com/btcsuite/btcwallet/walletdb v1.1.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk= github.com/btcsuite/btcwallet/wtxmgr v1.0.0 h1:aIHgViEmZmZfe0tQQqF1xyd2qBqFWxX5vZXkkbjtbeA= github.com/btcsuite/btcwallet/wtxmgr v1.0.0/go.mod h1:vc4gBprll6BP0UJ+AIGDaySoc7MdAmZf8kelfNb8CFY= github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 h1:kij1x2aL7VE6gtx8KMIt8PGPgI5GV9LgtHFG5KaEMPY= diff --git a/keychain/interface_test.go b/keychain/interface_test.go index 82e70e7c..a8571855 100644 --- a/keychain/interface_test.go +++ b/keychain/interface_test.go @@ -45,7 +45,7 @@ func createTestBtcWallet(coinType uint32) (func(), *wallet.Wallet, error) { if err != nil { return nil, nil, err } - loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, 0) + loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, true, 0) pass := []byte("test") diff --git a/lnd.go b/lnd.go index 9bf0426d..8c0b74aa 100644 --- a/lnd.go +++ b/lnd.go @@ -190,6 +190,7 @@ func Main(lisCfg ListenerCfg) error { graphDir, channeldb.OptionSetRejectCacheSize(cfg.Caches.RejectCacheSize), channeldb.OptionSetChannelCacheSize(cfg.Caches.ChannelCacheSize), + channeldb.OptionSetSyncFreelist(cfg.SyncFreelist), ) if err != nil { err := fmt.Errorf("Unable to open channeldb: %v", err) @@ -1000,7 +1001,8 @@ func waitForWalletPassword(restEndpoints []net.Addr, cfg.AdminMacPath, cfg.ReadMacPath, cfg.InvoiceMacPath, } pwService := walletunlocker.New( - chainConfig.ChainDir, activeNetParams.Params, macaroonFiles, + chainConfig.ChainDir, activeNetParams.Params, !cfg.SyncFreelist, + macaroonFiles, ) lnrpc.RegisterWalletUnlockerServer(grpcServer, pwService) @@ -1093,7 +1095,8 @@ func waitForWalletPassword(restEndpoints []net.Addr, chainConfig.ChainDir, activeNetParams.Params, ) loader := wallet.NewLoader( - activeNetParams.Params, netDir, uint32(recoveryWindow), + activeNetParams.Params, netDir, !cfg.SyncFreelist, + recoveryWindow, ) // With the seed, we can now use the wallet loader to create diff --git a/lnwallet/btcwallet/btcwallet.go b/lnwallet/btcwallet/btcwallet.go index 31c2e320..84945c86 100644 --- a/lnwallet/btcwallet/btcwallet.go +++ b/lnwallet/btcwallet/btcwallet.go @@ -89,8 +89,10 @@ func New(cfg Config) (*BtcWallet, error) { } else { pubPass = cfg.PublicPass } - loader := base.NewLoader(cfg.NetParams, netDir, - cfg.RecoveryWindow) + loader := base.NewLoader( + cfg.NetParams, netDir, cfg.NoFreelistSync, + cfg.RecoveryWindow, + ) walletExists, err := loader.WalletExists() if err != nil { return nil, err diff --git a/lnwallet/btcwallet/config.go b/lnwallet/btcwallet/config.go index 3516d5cb..2dedf4e3 100644 --- a/lnwallet/btcwallet/config.go +++ b/lnwallet/btcwallet/config.go @@ -75,6 +75,11 @@ type Config struct { // encrypted at all, in which case it should be attempted to be loaded // normally when creating the BtcWallet. Wallet *wallet.Wallet + + // NoFreelistSync, if true, prevents the database from syncing its + // freelist to disk, resulting in improved performance at the expense of + // increased startup time. + NoFreelistSync bool } // NetworkDir returns the directory name of a network directory to hold wallet diff --git a/lnwallet/interface_test.go b/lnwallet/interface_test.go index 99ef9b7b..04af443f 100644 --- a/lnwallet/interface_test.go +++ b/lnwallet/interface_test.go @@ -2804,8 +2804,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, // Start Alice - open a database, start a neutrino // instance, and initialize a btcwallet driver for it. - aliceDB, err := walletdb.Create("bdb", - tempTestDirAlice+"/neutrino.db") + aliceDB, err := walletdb.Create( + "bdb", tempTestDirAlice+"/neutrino.db", true, + ) if err != nil { t.Fatalf("unable to create DB: %v", err) } @@ -2831,8 +2832,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver, // Start Bob - open a database, start a neutrino // instance, and initialize a btcwallet driver for it. - bobDB, err := walletdb.Create("bdb", - tempTestDirBob+"/neutrino.db") + bobDB, err := walletdb.Create( + "bdb", tempTestDirBob+"/neutrino.db", true, + ) if err != nil { t.Fatalf("unable to create DB: %v", err) } diff --git a/routing/chainview/interface_test.go b/routing/chainview/interface_test.go index 24789b77..14f9461d 100644 --- a/routing/chainview/interface_test.go +++ b/routing/chainview/interface_test.go @@ -846,7 +846,7 @@ var interfaceImpls = []struct { } dbName := filepath.Join(spvDir, "neutrino.db") - spvDatabase, err := walletdb.Create("bdb", dbName) + spvDatabase, err := walletdb.Create("bdb", dbName, true) if err != nil { return nil, nil, err } diff --git a/walletunlocker/service.go b/walletunlocker/service.go index a4e030e0..90e84bae 100644 --- a/walletunlocker/service.go +++ b/walletunlocker/service.go @@ -96,13 +96,14 @@ type UnlockerService struct { // sent. UnlockMsgs chan *WalletUnlockMsg - chainDir string - netParams *chaincfg.Params - macaroonFiles []string + chainDir string + noFreelistSync bool + netParams *chaincfg.Params + macaroonFiles []string } // New creates and returns a new UnlockerService. -func New(chainDir string, params *chaincfg.Params, +func New(chainDir string, params *chaincfg.Params, noFreelistSync bool, macaroonFiles []string) *UnlockerService { return &UnlockerService{ @@ -128,7 +129,7 @@ func (u *UnlockerService) GenSeed(ctx context.Context, // Before we start, we'll ensure that the wallet hasn't already created // so we don't show a *new* seed to the user if one already exists. netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, 0) + loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0) walletExists, err := loader.WalletExists() if err != nil { return nil, err @@ -257,7 +258,9 @@ func (u *UnlockerService) InitWallet(ctx context.Context, // We'll then open up the directory that will be used to store the // wallet's files so we can check if the wallet already exists. netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, uint32(recoveryWindow)) + loader := wallet.NewLoader( + u.netParams, netDir, u.noFreelistSync, uint32(recoveryWindow), + ) walletExists, err := loader.WalletExists() if err != nil { @@ -314,7 +317,9 @@ func (u *UnlockerService) UnlockWallet(ctx context.Context, recoveryWindow := uint32(in.RecoveryWindow) netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, recoveryWindow) + loader := wallet.NewLoader( + u.netParams, netDir, u.noFreelistSync, recoveryWindow, + ) // Check if wallet already exists. walletExists, err := loader.WalletExists() @@ -365,7 +370,7 @@ func (u *UnlockerService) ChangePassword(ctx context.Context, in *lnrpc.ChangePasswordRequest) (*lnrpc.ChangePasswordResponse, error) { netDir := btcwallet.NetworkDir(u.chainDir, u.netParams) - loader := wallet.NewLoader(u.netParams, netDir, 0) + loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0) // First, we'll make sure the wallet exists for the specific chain and // network. diff --git a/walletunlocker/service_test.go b/walletunlocker/service_test.go index 8eec845a..9b497ede 100644 --- a/walletunlocker/service_test.go +++ b/walletunlocker/service_test.go @@ -36,7 +36,7 @@ var ( func createTestWallet(t *testing.T, dir string, netParams *chaincfg.Params) { netDir := btcwallet.NetworkDir(dir, netParams) - loader := wallet.NewLoader(netParams, netDir, 0) + loader := wallet.NewLoader(netParams, netDir, true, 0) _, err := loader.CreateNewWallet( testPassword, testPassword, testSeed, time.Time{}, ) @@ -62,7 +62,7 @@ func TestGenSeed(t *testing.T) { } defer os.RemoveAll(testDir) - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Now that the service has been created, we'll ask it to generate a // new seed for us given a test passphrase. @@ -103,7 +103,7 @@ func TestGenSeedGenerateEntropy(t *testing.T) { defer func() { os.RemoveAll(testDir) }() - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Now that the service has been created, we'll ask it to generate a // new seed for us given a test passphrase. Note that we don't actually @@ -143,7 +143,7 @@ func TestGenSeedInvalidEntropy(t *testing.T) { defer func() { os.RemoveAll(testDir) }() - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Now that the service has been created, we'll ask it to generate a // new seed for us given a test passphrase. However, we'll be using an @@ -181,7 +181,7 @@ func TestInitWallet(t *testing.T) { }() // Create new UnlockerService. - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // Once we have the unlocker service created, we'll now instantiate a // new cipher seed instance. @@ -282,7 +282,7 @@ func TestCreateWalletInvalidEntropy(t *testing.T) { }() // Create new UnlockerService. - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) // We'll attempt to init the wallet with an invalid cipher seed and // passphrase. @@ -315,7 +315,7 @@ func TestUnlockWallet(t *testing.T) { }() // Create new UnlockerService. - service := walletunlocker.New(testDir, testNetParams, nil) + service := walletunlocker.New(testDir, testNetParams, true, nil) ctx := context.Background() req := &lnrpc.UnlockWalletRequest{ @@ -389,7 +389,7 @@ func TestChangeWalletPassword(t *testing.T) { } // Create a new UnlockerService with our temp files. - service := walletunlocker.New(testDir, testNetParams, tempFiles) + service := walletunlocker.New(testDir, testNetParams, true, tempFiles) ctx := context.Background() newPassword := []byte("hunter2???")