multi: support sync freelist option within btcwallet

chainntnfs/test_utils.go

@@ -268,7 +268,7 @@ func NewNeutrinoBackend(t *testing.T, minerAddr string) (*neutrino.ChainService,
 	}
 
 	dbName := filepath.Join(spvDir, "neutrino.db")
-	spvDatabase, err := walletdb.Create("bdb", dbName)
+	spvDatabase, err := walletdb.Create("bdb", dbName, true)
 	if err != nil {
 		os.RemoveAll(spvDir)
 		t.Fatalf("unable to create walletdb: %v", err)

chainregistry.go

@@ -703,7 +703,7 @@ func initNeutrinoBackend(chainDir string) (*neutrino.ChainService, func(), error
 	}
 
 	dbName := filepath.Join(dbPath, "neutrino.db")
-	db, err := walletdb.Create("bdb", dbName)
+	db, err := walletdb.Create("bdb", dbName, true)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to create neutrino "+
 			"database: %v", err)

go.mod

@@ -8,10 +8,10 @@ require (
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
-	github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953
+	github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb
 	github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0
 	github.com/btcsuite/btcwallet/wallet/txrules v1.0.0
-	github.com/btcsuite/btcwallet/walletdb v1.0.0
+	github.com/btcsuite/btcwallet/walletdb v1.1.0
 	github.com/btcsuite/btcwallet/wtxmgr v1.0.0
 	github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941
 	github.com/coreos/bbolt v1.3.3

go.sum

@@ -25,8 +25,8 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d h1:yJzD/yFppdVCf6ApMkVy8cUxV0XrxdP9rVf6D87/Mng=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
-github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953 h1:NG3SmXd3KMOF4/BHVQaJuayrlXBosJgwUjeHcX4k198=
+github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb h1:W4l92i/ImG2qx4msNWSzhKqnvfQM3d8weQ27D++lhxk=
-github.com/btcsuite/btcwallet v0.0.0-20190925005052-95d7aa0b4953/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI=
+github.com/btcsuite/btcwallet v0.0.0-20191004044542-5f205d607bcb/go.mod h1:ntLqUbZ12G8FmPX1nJj7W83WiAFOLRGiuarH4zDYdlI=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 h1:KGHMW5sd7yDdDMkCZ/JpP0KltolFsQcB973brBnfj4c=
 github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0/go.mod h1:VufDts7bd/zs3GV13f/lXc/0lXrPnvxD/NvmpG/FEKU=
 github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 h1:2VsfS0sBedcM5KmDzRMT3+b6xobqWveZGvjb+jFez5w=
@@ -35,6 +35,8 @@ github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0 h1:6DxkcoMnCPY4E9cUDPB5tbuuf
 github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0/go.mod h1:pauEU8UuMFiThe5PB3EO+gO5kx87Me5NvdQDsTuq6cs=
 github.com/btcsuite/btcwallet/walletdb v1.0.0 h1:mheT7vCWK5EP6rZzhxsQ7ms9+yX4VE8bwiJctECBeNw=
 github.com/btcsuite/btcwallet/walletdb v1.0.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
+github.com/btcsuite/btcwallet/walletdb v1.1.0 h1:JHAL7wZ8pX4SULabeAv/wPO9sseRWMGzE80lfVmRw6Y=
+github.com/btcsuite/btcwallet/walletdb v1.1.0/go.mod h1:bZTy9RyYZh9fLnSua+/CD48TJtYJSHjjYcSaszuxCCk=
 github.com/btcsuite/btcwallet/wtxmgr v1.0.0 h1:aIHgViEmZmZfe0tQQqF1xyd2qBqFWxX5vZXkkbjtbeA=
 github.com/btcsuite/btcwallet/wtxmgr v1.0.0/go.mod h1:vc4gBprll6BP0UJ+AIGDaySoc7MdAmZf8kelfNb8CFY=
 github.com/btcsuite/fastsha256 v0.0.0-20160815193821-637e65642941 h1:kij1x2aL7VE6gtx8KMIt8PGPgI5GV9LgtHFG5KaEMPY=

keychain/interface_test.go

@@ -45,7 +45,7 @@ func createTestBtcWallet(coinType uint32) (func(), *wallet.Wallet, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, 0)
+	loader := wallet.NewLoader(&chaincfg.SimNetParams, tempDir, true, 0)
 
 	pass := []byte("test")
 

lnd.go

@@ -1000,7 +1000,8 @@ func waitForWalletPassword(restEndpoints []net.Addr,
 		cfg.AdminMacPath, cfg.ReadMacPath, cfg.InvoiceMacPath,
 	}
 	pwService := walletunlocker.New(
-		chainConfig.ChainDir, activeNetParams.Params, macaroonFiles,
+		chainConfig.ChainDir, activeNetParams.Params, true,
+		macaroonFiles,
 	)
 	lnrpc.RegisterWalletUnlockerServer(grpcServer, pwService)
 
@@ -1093,7 +1094,7 @@ func waitForWalletPassword(restEndpoints []net.Addr,
 			chainConfig.ChainDir, activeNetParams.Params,
 		)
 		loader := wallet.NewLoader(
-			activeNetParams.Params, netDir, uint32(recoveryWindow),
+			activeNetParams.Params, netDir, true, recoveryWindow,
 		)
 
 		// With the seed, we can now use the wallet loader to create

lnwallet/btcwallet/btcwallet.go

@@ -89,8 +89,10 @@ func New(cfg Config) (*BtcWallet, error) {
 		} else {
 			pubPass = cfg.PublicPass
 		}
-		loader := base.NewLoader(cfg.NetParams, netDir,
+		loader := base.NewLoader(
-			cfg.RecoveryWindow)
+			cfg.NetParams, netDir, cfg.NoFreelistSync,
+			cfg.RecoveryWindow,
+		)
 		walletExists, err := loader.WalletExists()
 		if err != nil {
 			return nil, err

lnwallet/btcwallet/config.go

@@ -75,6 +75,11 @@ type Config struct {
 	// encrypted at all, in which case it should be attempted to be loaded
 	// normally when creating the BtcWallet.
 	Wallet *wallet.Wallet
+
+	// NoFreelistSync, if true, prevents the database from syncing its
+	// freelist to disk, resulting in improved performance at the expense of
+	// increased startup time.
+	NoFreelistSync bool
 }
 
 // NetworkDir returns the directory name of a network directory to hold wallet

lnwallet/interface_test.go

@@ -2804,8 +2804,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 
 			// Start Alice - open a database, start a neutrino
 			// instance, and initialize a btcwallet driver for it.
-			aliceDB, err := walletdb.Create("bdb",
+			aliceDB, err := walletdb.Create(
-				tempTestDirAlice+"/neutrino.db")
+				"bdb", tempTestDirAlice+"/neutrino.db", true,
+			)
 			if err != nil {
 				t.Fatalf("unable to create DB: %v", err)
 			}
@@ -2831,8 +2832,9 @@ func runTests(t *testing.T, walletDriver *lnwallet.WalletDriver,
 
 			// Start Bob - open a database, start a neutrino
 			// instance, and initialize a btcwallet driver for it.
-			bobDB, err := walletdb.Create("bdb",
+			bobDB, err := walletdb.Create(
-				tempTestDirBob+"/neutrino.db")
+				"bdb", tempTestDirBob+"/neutrino.db", true,
+			)
 			if err != nil {
 				t.Fatalf("unable to create DB: %v", err)
 			}

routing/chainview/interface_test.go

@@ -846,7 +846,7 @@ var interfaceImpls = []struct {
 			}
 
 			dbName := filepath.Join(spvDir, "neutrino.db")
-			spvDatabase, err := walletdb.Create("bdb", dbName)
+			spvDatabase, err := walletdb.Create("bdb", dbName, true)
 			if err != nil {
 				return nil, nil, err
 			}

walletunlocker/service.go

@@ -96,13 +96,14 @@ type UnlockerService struct {
 	// sent.
 	UnlockMsgs chan *WalletUnlockMsg
 
-	chainDir      string
+	chainDir       string
-	netParams     *chaincfg.Params
+	noFreelistSync bool
-	macaroonFiles []string
+	netParams      *chaincfg.Params
+	macaroonFiles  []string
 }
 
 // New creates and returns a new UnlockerService.
-func New(chainDir string, params *chaincfg.Params,
+func New(chainDir string, params *chaincfg.Params, noFreelistSync bool,
 	macaroonFiles []string) *UnlockerService {
 
 	return &UnlockerService{
@@ -128,7 +129,7 @@ func (u *UnlockerService) GenSeed(ctx context.Context,
 	// Before we start, we'll ensure that the wallet hasn't already created
 	// so we don't show a *new* seed to the user if one already exists.
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, 0)
+	loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0)
 	walletExists, err := loader.WalletExists()
 	if err != nil {
 		return nil, err
@@ -257,7 +258,9 @@ func (u *UnlockerService) InitWallet(ctx context.Context,
 	// We'll then open up the directory that will be used to store the
 	// wallet's files so we can check if the wallet already exists.
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, uint32(recoveryWindow))
+	loader := wallet.NewLoader(
+		u.netParams, netDir, u.noFreelistSync, uint32(recoveryWindow),
+	)
 
 	walletExists, err := loader.WalletExists()
 	if err != nil {
@@ -314,7 +317,9 @@ func (u *UnlockerService) UnlockWallet(ctx context.Context,
 	recoveryWindow := uint32(in.RecoveryWindow)
 
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, recoveryWindow)
+	loader := wallet.NewLoader(
+		u.netParams, netDir, u.noFreelistSync, recoveryWindow,
+	)
 
 	// Check if wallet already exists.
 	walletExists, err := loader.WalletExists()
@@ -365,7 +370,7 @@ func (u *UnlockerService) ChangePassword(ctx context.Context,
 	in *lnrpc.ChangePasswordRequest) (*lnrpc.ChangePasswordResponse, error) {
 
 	netDir := btcwallet.NetworkDir(u.chainDir, u.netParams)
-	loader := wallet.NewLoader(u.netParams, netDir, 0)
+	loader := wallet.NewLoader(u.netParams, netDir, u.noFreelistSync, 0)
 
 	// First, we'll make sure the wallet exists for the specific chain and
 	// network.

walletunlocker/service_test.go

@@ -36,7 +36,7 @@ var (
 
 func createTestWallet(t *testing.T, dir string, netParams *chaincfg.Params) {
 	netDir := btcwallet.NetworkDir(dir, netParams)
-	loader := wallet.NewLoader(netParams, netDir, 0)
+	loader := wallet.NewLoader(netParams, netDir, true, 0)
 	_, err := loader.CreateNewWallet(
 		testPassword, testPassword, testSeed, time.Time{},
 	)
@@ -62,7 +62,7 @@ func TestGenSeed(t *testing.T) {
 	}
 	defer os.RemoveAll(testDir)
 
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Now that the service has been created, we'll ask it to generate a
 	// new seed for us given a test passphrase.
@@ -103,7 +103,7 @@ func TestGenSeedGenerateEntropy(t *testing.T) {
 	defer func() {
 		os.RemoveAll(testDir)
 	}()
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Now that the service has been created, we'll ask it to generate a
 	// new seed for us given a test passphrase. Note that we don't actually
@@ -143,7 +143,7 @@ func TestGenSeedInvalidEntropy(t *testing.T) {
 	defer func() {
 		os.RemoveAll(testDir)
 	}()
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Now that the service has been created, we'll ask it to generate a
 	// new seed for us given a test passphrase. However, we'll be using an
@@ -181,7 +181,7 @@ func TestInitWallet(t *testing.T) {
 	}()
 
 	// Create new UnlockerService.
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// Once we have the unlocker service created, we'll now instantiate a
 	// new cipher seed instance.
@@ -282,7 +282,7 @@ func TestCreateWalletInvalidEntropy(t *testing.T) {
 	}()
 
 	// Create new UnlockerService.
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	// We'll attempt to init the wallet with an invalid cipher seed and
 	// passphrase.
@@ -315,7 +315,7 @@ func TestUnlockWallet(t *testing.T) {
 	}()
 
 	// Create new UnlockerService.
-	service := walletunlocker.New(testDir, testNetParams, nil)
+	service := walletunlocker.New(testDir, testNetParams, true, nil)
 
 	ctx := context.Background()
 	req := &lnrpc.UnlockWalletRequest{
@@ -389,7 +389,7 @@ func TestChangeWalletPassword(t *testing.T) {
 	}
 
 	// Create a new UnlockerService with our temp files.
-	service := walletunlocker.New(testDir, testNetParams, tempFiles)
+	service := walletunlocker.New(testDir, testNetParams, true, tempFiles)
 
 	ctx := context.Background()
 	newPassword := []byte("hunter2???")