multi: add AutoCompact option to bolt backend

With this commit we thread the new AutoCompact flags all the way through
to the bolt backend.

channeldb/db.go

@@ -238,7 +238,13 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {
 		modifier(&opts)
 	}
 
-	backend, err := kvdb.GetBoltBackend(dbPath, dbName, opts.NoFreelistSync)
+	backend, err := kvdb.GetBoltBackend(&kvdb.BoltBackendConfig{
+		DBPath:            dbPath,
+		DBFileName:        dbName,
+		NoFreelistSync:    opts.NoFreelistSync,
+		AutoCompact:       opts.AutoCompact,
+		AutoCompactMinAge: opts.AutoCompactMinAge,
+	})
 	if err != nil {
 		return nil, err
 	}

channeldb/kvdb/backend.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	_ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend.
 )
@@ -19,25 +20,53 @@ func fileExists(path string) bool {
 	return true
 }
 
+// BoltBackendConfig is a struct that holds settings specific to the bolt
+// database backend.
+type BoltBackendConfig struct {
+	// DBPath is the directory path in which the database file should be
+	// stored.
+	DBPath string
+
+	// DBFileName is the name of the database file.
+	DBFileName string
+
+	// NoFreelistSync, if true, prevents the database from syncing its
+	// freelist to disk, resulting in improved performance at the expense of
+	// increased startup time.
+	NoFreelistSync bool
+
+	// AutoCompact specifies if a Bolt based database backend should be
+	// automatically compacted on startup (if the minimum age of the
+	// database file is reached). This will require additional disk space
+	// for the compacted copy of the database but will result in an overall
+	// lower database size after the compaction.
+	AutoCompact bool
+
+	// AutoCompactMinAge specifies the minimum time that must have passed
+	// since a bolt database file was last compacted for the compaction to
+	// be considered again.
+	AutoCompactMinAge time.Duration
+}
+
 // GetBoltBackend opens (or creates if doesn't exits) a bbolt
 // backed database and returns a kvdb.Backend wrapping it.
-func GetBoltBackend(path, name string, noFreeListSync bool) (Backend, error) {
+func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) {
-	dbFilePath := filepath.Join(path, name)
+	dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName)
 	var (
 		db  Backend
 		err error
 	)
 
 	if !fileExists(dbFilePath) {
-		if !fileExists(path) {
+		if !fileExists(cfg.DBPath) {
-			if err := os.MkdirAll(path, 0700); err != nil {
+			if err := os.MkdirAll(cfg.DBPath, 0700); err != nil {
 				return nil, err
 			}
 		}
 
-		db, err = Create(BoltBackendName, dbFilePath, noFreeListSync)
+		db, err = Create(BoltBackendName, dbFilePath, cfg.NoFreelistSync)
 	} else {
-		db, err = Open(BoltBackendName, dbFilePath, noFreeListSync)
+		db, err = Open(BoltBackendName, dbFilePath, cfg.NoFreelistSync)
 	}
 
 	if err != nil {
@@ -57,7 +86,11 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 	empty := func() {}
 
 	if TestBackend == BoltBackendName {
-		db, err := GetBoltBackend(path, name, true)
+		db, err := GetBoltBackend(&BoltBackendConfig{
+			DBPath:         path,
+			DBFileName:     name,
+			NoFreelistSync: true,
+		})
 		if err != nil {
 			return nil, nil, err
 		}

channeldb/options.go

@@ -1,6 +1,11 @@
 package channeldb
 
-import "github.com/lightningnetwork/lnd/clock"
+import (
+	"time"
+
+	"github.com/lightningnetwork/lnd/channeldb/kvdb"
+	"github.com/lightningnetwork/lnd/clock"
+)
 
 const (
 	// DefaultRejectCacheSize is the default number of rejectCacheEntries to
@@ -16,6 +21,8 @@ const (
 
 // Options holds parameters for tuning and customizing a channeldb.DB.
 type Options struct {
+	kvdb.BoltBackendConfig
+
 	// RejectCacheSize is the maximum number of rejectCacheEntries to hold
 	// in the rejection cache.
 	RejectCacheSize int
@@ -24,11 +31,6 @@ type Options struct {
 	// channel cache.
 	ChannelCacheSize int
 
-	// NoFreelistSync, if true, prevents the database from syncing its
-	// freelist to disk, resulting in improved performance at the expense of
-	// increased startup time.
-	NoFreelistSync bool
-
 	// clock is the time source used by the database.
 	clock clock.Clock
 
@@ -40,9 +42,13 @@ type Options struct {
 // DefaultOptions returns an Options populated with default values.
 func DefaultOptions() Options {
 	return Options{
+		BoltBackendConfig: kvdb.BoltBackendConfig{
+			NoFreelistSync:    true,
+			AutoCompact:       false,
+			AutoCompactMinAge: kvdb.DefaultBoltAutoCompactMinAge,
+		},
 		RejectCacheSize:  DefaultRejectCacheSize,
 		ChannelCacheSize: DefaultChannelCacheSize,
-		NoFreelistSync:   true,
 		clock:            clock.NewDefaultClock(),
 	}
 }
@@ -71,6 +77,21 @@ func OptionSetSyncFreelist(b bool) OptionModifier {
 	}
 }
 
+// OptionAutoCompact turns on automatic database compaction on startup.
+func OptionAutoCompact() OptionModifier {
+	return func(o *Options) {
+		o.AutoCompact = true
+	}
+}
+
+// OptionAutoCompactMinAge sets the minimum age for automatic database
+// compaction.
+func OptionAutoCompactMinAge(minAge time.Duration) OptionModifier {
+	return func(o *Options) {
+		o.AutoCompactMinAge = minAge
+	}
+}
+
 // OptionClock sets a non-default clock dependency.
 func OptionClock(clock clock.Clock) OptionModifier {
 	return func(o *Options) {

lncfg/db.go

@@ -83,9 +83,13 @@ func (db *DB) GetBackends(ctx context.Context, dbPath string,
 		}
 	}
 
-	localDB, err = kvdb.GetBoltBackend(
+	localDB, err = kvdb.GetBoltBackend(&kvdb.BoltBackendConfig{
-		dbPath, dbName, !db.Bolt.SyncFreelist,
+		DBPath:            dbPath,
-	)
+		DBFileName:        dbName,
+		NoFreelistSync:    !db.Bolt.SyncFreelist,
+		AutoCompact:       db.Bolt.AutoCompact,
+		AutoCompactMinAge: db.Bolt.AutoCompactMinAge,
+	})
 	if err != nil {
 		return nil, err
 	}

lnd.go

@@ -1353,8 +1353,9 @@ func initializeDatabases(ctx context.Context,
 		"minutes...")
 
 	if cfg.DB.Backend == lncfg.BoltBackend {
-		ltndLog.Infof("Opening bbolt database, sync_freelist=%v",
+		ltndLog.Infof("Opening bbolt database, sync_freelist=%v, "+
-			cfg.DB.Bolt.SyncFreelist)
+			"auto_compact=%v", cfg.DB.Bolt.SyncFreelist,
+			cfg.DB.Bolt.AutoCompact)
 	}
 
 	startOpenTime := time.Now()