Merge pull request #3578 from wpaulino/no-freelist-sync-option

config: add sync freelist cli flag

chainntnfs/test_utils.go

@@ -268,7 +268,7 @@ func NewNeutrinoBackend(t *testing.T, minerAddr string) (*neutrino.ChainService,
 	}
 
 	dbName := filepath.Join(spvDir, "neutrino.db")
-	spvDatabase, err := walletdb.Create("bdb", dbName)
+	spvDatabase, err := walletdb.Create("bdb", dbName, true)
 	if err != nil {
 		os.RemoveAll(spvDir)
 		t.Fatalf("unable to create walletdb: %v", err)

chainregistry.go

@@ -703,7 +703,7 @@ func initNeutrinoBackend(chainDir string) (*neutrino.ChainService, func(), error
 	}
 
 	dbName := filepath.Join(dbPath, "neutrino.db")
-	db, err := walletdb.Create("bdb", dbName)
+	db, err := walletdb.Create("bdb", dbName, !cfg.SyncFreelist)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to create neutrino "+
 			"database: %v", err)

channeldb/db.go

@@ -85,7 +85,7 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {
 	// Specify bbolt freelist options to reduce heap pressure in case the
 	// freelist grows to be very large.
 	options := &bbolt.Options{
-		NoFreelistSync: true,
+		NoFreelistSync: opts.NoFreelistSync,
 		FreelistType:   bbolt.FreelistMapType,
 	}
 

channeldb/options.go

@@ -21,6 +21,11 @@ type Options struct {
 	// ChannelCacheSize is the maximum number of ChannelEdges to hold in the
 	// channel cache.
 	ChannelCacheSize int
+
+	// NoFreelistSync, if true, prevents the database from syncing its
+	// freelist to disk, resulting in improved performance at the expense of
+	// increased startup time.
+	NoFreelistSync bool
 }
 
 // DefaultOptions returns an Options populated with default values.
@@ -28,6 +33,7 @@ func DefaultOptions() Options {
 	return Options{
 		RejectCacheSize:  DefaultRejectCacheSize,
 		ChannelCacheSize: DefaultChannelCacheSize,
+		NoFreelistSync:   true,
 	}
 }
 
@@ -47,3 +53,10 @@ func OptionSetChannelCacheSize(n int) OptionModifier {
 		o.ChannelCacheSize = n
 	}
 }
+
+// OptionSetSyncFreelist allows the database to sync its freelist.
+func OptionSetSyncFreelist(b bool) OptionModifier {
+	return func(o *Options) {
+		o.NoFreelistSync = !b
+	}
+}

config.go

@@ -234,6 +234,7 @@ type config struct {
 	LndDir          string   `long:"lnddir" description:"The base directory that contains lnd's data, logs, configuration file, etc."`
 	ConfigFile      string   `short:"C" long:"configfile" description:"Path to configuration file"`
 	DataDir         string   `short:"b" long:"datadir" description:"The directory to store lnd's data within"`
+	SyncFreelist    bool     `long:"sync-freelist" description:"Whether the databases used within lnd should sync their freelist to disk. This is disabled by default resulting in improved memory performance during operation, but with an increase in startup time."`
 	TLSCertPath     string   `long:"tlscertpath" description:"Path to write the TLS certificate for lnd's RPC and REST services"`
 	TLSKeyPath      string   `long:"tlskeypath" description:"Path to write the TLS private key for lnd's RPC and REST services"`
 	TLSExtraIPs     []string `long:"tlsextraip" description:"Adds an extra ip to the generated certificate"`

go.mod

@@ -8,10 +8,10 @@ require (
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
-	github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953
+	github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0
 	github.com/btcsuite/btcwallet/wallet/txrules v1.0.0
-	github.com/btcsuite/btcwallet/walletdb v1.0.0
+	github.com/btcsuite/btcwallet/walletdb v1.1.0
 	github.com/btcsuite/btcwallet/wtxmgr v1.0.0
 	github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941
 	github.com/coreos/bbolt v1.3.3

go.sum

@@ -25,8 +25,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953 h1:NG3SmXd3KMOF4/BHVQaJuayrlXBosJgwUjeHcX4k198=
+github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb h1:W4l92i/ImG2qx4msNWSzhKqnvfQM3d8weQ27D++lhxk=
-github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI=
+github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 h1:KGHMW5sd7yDdDMkCZ/JpP0KltolFsQcB973brBnfj4c=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU=
 github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 h1:2VsfS0sBedcM5KmDzRMT3+b6xobqWveZGvjb+jFez5w=
@@ -35,6 +35,8 @@ github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0 h1:6DxkcoMnCPY4E9cUDPB5tbuuf
 github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs=
 github.com/btcsuite/btcwallet/walletdb v1.0.0 h1:mheT7vCWK5EP6rZzhxsQ7ms9+yX4VE8bwiJctECBeNw=
 github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
+github.com/btcsuite/btcwallet/walletdb v1.1.0 h1:JHAL7wZ8pX4SULabeAv/wPO9sseRWMGzE80lfVmRw6Y=
+github.com/btcsuite/btcwallet/walletdb v1.1.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
 github.com/btcsuite/btcwallet/wtxmgr v1.0.0 h1:aIHgViEmZmZfe0tQQqF1xyd2qBqFWxX5vZXkkbjtbeA=
 github.com/btcsuite/btcwallet/wtxmgr v1.0.0/go.mod h1:vc4gBprll6BP0UJ+AIGDaySoc7MdAmZf8kelfNb8CFY=
 github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 h1:kij1x2aL7VE6gtx8KMIt8PGPgI5GV9LgtHFG5KaEMPY=

keychain/interface_test.go

@@ -45,7 +45,7 @@ func createTestBtcWallet(coinType uint32) (func(), *wallet.Wallet, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, 0)
+	loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, true, 0)
 
 	pass := []byte("test")
 

lnd.go

@@ -190,6 +190,7 @@ func Main(lisCfg ListenerCfg) error {
 		graphDir,
 		channeldb.OptionSetRejectCacheSize(cfg.Caches.RejectCacheSize),
 		channeldb.OptionSetChannelCacheSize(cfg.Caches.ChannelCacheSize),
+		channeldb.OptionSetSyncFreelist(cfg.SyncFreelist),
 	)
 	if err != nil {
 		err := fmt.Errorf("Unable to open channeldb: %v", err)
@@ -1000,7 +1001,8 @@ func waitForWalletPassword(restEndpoints []net.Addr,
 		cfg.AdminMacPath, cfg.ReadMacPath, cfg.InvoiceMacPath,
 	}
 	pwService := walletunlocker.New(
-		chainConfig.ChainDir, activeNetParams.Params, macaroonFiles,
+		chainConfig.ChainDir, activeNetParams.Params, !cfg.SyncFreelist,
+		macaroonFiles,
 	)
 	lnrpc.RegisterWalletUnlockerServer(grpcServer, pwService)
 
@@ -1093,7 +1095,8 @@ func waitForWalletPassword(restEndpoints []net.Addr,
 			chainConfig.ChainDir, activeNetParams.Params,
 		)
 		loader := wallet.NewLoader(
-			activeNetParams.Params, netDir, uint32(recoveryWindow),
+			activeNetParams.Params, netDir, !cfg.SyncFreelist,
+			recoveryWindow,
 		)
 
 		// With the seed, we can now use the wallet loader to create

lnwallet/btcwallet/btcwallet.go

@@ -89,8 +89,10 @@ func New(cfg Config) (*BtcWallet, error) {
 		} else {
 			pubPass = cfg.PublicPass
 		}
-		loader := base.NewLoader(cfg.NetParams, netDir,
+		loader := base.NewLoader(
-			cfg.RecoveryWindow)
+			cfg.NetParams, netDir, cfg.NoFreelistSync,
+			cfg.RecoveryWindow,
+		)
 		walletExists, err := loader.WalletExists()
 		if err != nil {
 			return nil, err

lnwallet/btcwallet/config.go

@@ -75,6 +75,11 @@ type Config struct {
 	// encrypted at all, in which case it should be attempted to be loaded
 	// normally when creating the BtcWallet.
 	Wallet *wallet.Wallet
+
+	// NoFreelistSync, if true, prevents the database from syncing its
+	// freelist to disk, resulting in improved performance at the expense of
+	// increased startup time.
+	NoFreelistSync bool
 }
 
 // NetworkDir returns the directory name of a network directory to hold wallet

lnwallet/interface_test.go

@@ -2804,8 +2804,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 
 			// Start Alice - open a database, start a neutrino
 			// instance, and initialize a btcwallet driver for it.
-			aliceDB, err := walletdb.Create("bdb",
+			aliceDB, err := walletdb.Create(
-				tempTestDirAlice+"/neutrino.db")
+				"bdb", tempTestDirAlice+"/neutrino.db", true,
+			)
 			if err != nil {
 				t.Fatalf("unable to create DB: %v", err)
 			}
@@ -2831,8 +2832,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 
 			// Start Bob - open a database, start a neutrino
 			// instance, and initialize a btcwallet driver for it.
-			bobDB, err := walletdb.Create("bdb",
+			bobDB, err := walletdb.Create(
-				tempTestDirBob+"/neutrino.db")
+				"bdb", tempTestDirBob+"/neutrino.db", true,
+			)
 			if err != nil {
 				t.Fatalf("unable to create DB: %v", err)
 			}

routing/chainview/interface_test.go

@@ -846,7 +846,7 @@ var interfaceImpls = []struct {
 			}
 
 			dbName := filepath.Join(spvDir, "neutrino.db")
-			spvDatabase, err := walletdb.Create("bdb", dbName)
+			spvDatabase, err := walletdb.Create("bdb", dbName, true)
 			if err != nil {
 				return nil, nil, err
 			}

walletunlocker/service.go

@@ -96,13 +96,14 @@ type UnlockerService struct {
 	// sent.
 	UnlockMsgs chan *WalletUnlockMsg
 
-	chainDir      string
+	chainDir       string
-	netParams     *chaincfg.Params
+	noFreelistSync bool
-	macaroonFiles []string
+	netParams      *chaincfg.Params
+	macaroonFiles  []string
 }
 
 // New creates and returns a new UnlockerService.
-func New(chainDir string, params *chaincfg.Params,
+func New(chainDir string, params *chaincfg.Params, noFreelistSync bool,
 	macaroonFiles []string) *UnlockerService {
 
 	return &UnlockerService{
@@ -128,7 +129,7 @@ func (u *UnlockerService) GenSeed(ctx context.Context,
 	// Before we start, we'll ensure that the wallet hasn't already created
 	// so we don't show a *new* seed to the user if one already exists.
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, 0)
+	loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0)
 	walletExists, err := loader.WalletExists()
 	if err != nil {
 		return nil, err
@@ -257,7 +258,9 @@ func (u *UnlockerService) InitWallet(ctx context.Context,
 	// We'll then open up the directory that will be used to store the
 	// wallet's files so we can check if the wallet already exists.
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, uint32(recoveryWindow))
+	loader := wallet.NewLoader(
+		u.netParams, netDir, u.noFreelistSync, uint32(recoveryWindow),
+	)
 
 	walletExists, err := loader.WalletExists()
 	if err != nil {
@@ -314,7 +317,9 @@ func (u *UnlockerService) UnlockWallet(ctx context.Context,
 	recoveryWindow := uint32(in.RecoveryWindow)
 
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, recoveryWindow)
+	loader := wallet.NewLoader(
+		u.netParams, netDir, u.noFreelistSync, recoveryWindow,
+	)
 
 	// Check if wallet already exists.
 	walletExists, err := loader.WalletExists()
@@ -365,7 +370,7 @@ func (u *UnlockerService) ChangePassword(ctx context.Context,
 	in *lnrpc.ChangePasswordRequest) (*lnrpc.ChangePasswordResponse, error) {
 
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, 0)
+	loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0)
 
 	// First, we'll make sure the wallet exists for the specific chain and
 	// network.

walletunlocker/service_test.go

@@ -36,7 +36,7 @@ var (
 
 func createTestWallet(t *testing.T, dir string, netParams *chaincfg.Params) {
 	netDir := btcwallet.NetworkDir(dir, netParams)
-	loader := wallet.NewLoader(netParams, netDir, 0)
+	loader := wallet.NewLoader(netParams, netDir, true, 0)
 	_, err := loader.CreateNewWallet(
 		testPassword, testPassword, testSeed, time.Time{},
 	)
@@ -62,7 +62,7 @@ func TestGenSeed(t *testing.T) {
 	}
 	defer os.RemoveAll(testDir)
 
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Now that the service has been created, we'll ask it to generate a
 	// new seed for us given a test passphrase.
@@ -103,7 +103,7 @@ func TestGenSeedGenerateEntropy(t *testing.T) {
 	defer func() {
 		os.RemoveAll(testDir)
 	}()
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Now that the service has been created, we'll ask it to generate a
 	// new seed for us given a test passphrase. Note that we don't actually
@@ -143,7 +143,7 @@ func TestGenSeedInvalidEntropy(t *testing.T) {
 	defer func() {
 		os.RemoveAll(testDir)
 	}()
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Now that the service has been created, we'll ask it to generate a
 	// new seed for us given a test passphrase. However, we'll be using an
@@ -181,7 +181,7 @@ func TestInitWallet(t *testing.T) {
 	}()
 
 	// Create new UnlockerService.
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Once we have the unlocker service created, we'll now instantiate a
 	// new cipher seed instance.
@@ -282,7 +282,7 @@ func TestCreateWalletInvalidEntropy(t *testing.T) {
 	}()
 
 	// Create new UnlockerService.
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// We'll attempt to init the wallet with an invalid cipher seed and
 	// passphrase.
@@ -315,7 +315,7 @@ func TestUnlockWallet(t *testing.T) {
 	}()
 
 	// Create new UnlockerService.
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	ctx := context.Background()
 	req := &lnrpc.UnlockWalletRequest{
@@ -389,7 +389,7 @@ func TestChangeWalletPassword(t *testing.T) {
 	}
 
 	// Create a new UnlockerService with our temp files.
-	service := walletunlocker.New(testDir, testNetParams, tempFiles)
+	service := walletunlocker.New(testDir, testNetParams, true, tempFiles)
 
 	ctx := context.Background()
 	newPassword := []byte("hunter2???")