From f8907fdb47daaab90923827d2b9d01eeaf235776 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:25 +0100 Subject: [PATCH] multi: add AutoCompact option to bolt backend With this commit we thread the new AutoCompact flags all the way through to the bolt backend. --- channeldb/db.go | 8 ++++++- channeldb/kvdb/backend.go | 47 +++++++++++++++++++++++++++++++++------ channeldb/options.go | 35 +++++++++++++++++++++++------ lncfg/db.go | 10 ++++++--- lnd.go | 5 +++-- 5 files changed, 85 insertions(+), 20 deletions(-) diff --git a/channeldb/db.go b/channeldb/db.go index 018c01b2..465003c4 100644 --- a/channeldb/db.go +++ b/channeldb/db.go @@ -238,7 +238,13 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) { modifier(&opts) } - backend, err := kvdb.GetBoltBackend(dbPath, dbName, opts.NoFreelistSync) + backend, err := kvdb.GetBoltBackend(&kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: dbName, + NoFreelistSync: opts.NoFreelistSync, + AutoCompact: opts.AutoCompact, + AutoCompactMinAge: opts.AutoCompactMinAge, + }) if err != nil { return nil, err } diff --git a/channeldb/kvdb/backend.go b/channeldb/kvdb/backend.go index 5f710ed9..0acedcb6 100644 --- a/channeldb/kvdb/backend.go +++ b/channeldb/kvdb/backend.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "time" _ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend. ) @@ -19,25 +20,53 @@ func fileExists(path string) bool { return true } +// BoltBackendConfig is a struct that holds settings specific to the bolt +// database backend. +type BoltBackendConfig struct { + // DBPath is the directory path in which the database file should be + // stored. + DBPath string + + // DBFileName is the name of the database file. + DBFileName string + + // NoFreelistSync, if true, prevents the database from syncing its + // freelist to disk, resulting in improved performance at the expense of + // increased startup time. + NoFreelistSync bool + + // AutoCompact specifies if a Bolt based database backend should be + // automatically compacted on startup (if the minimum age of the + // database file is reached). This will require additional disk space + // for the compacted copy of the database but will result in an overall + // lower database size after the compaction. + AutoCompact bool + + // AutoCompactMinAge specifies the minimum time that must have passed + // since a bolt database file was last compacted for the compaction to + // be considered again. + AutoCompactMinAge time.Duration +} + // GetBoltBackend opens (or creates if doesn't exits) a bbolt // backed database and returns a kvdb.Backend wrapping it. -func GetBoltBackend(path, name string, noFreeListSync bool) (Backend, error) { - dbFilePath := filepath.Join(path, name) +func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) { + dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName) var ( db Backend err error ) if !fileExists(dbFilePath) { - if !fileExists(path) { - if err := os.MkdirAll(path, 0700); err != nil { + if !fileExists(cfg.DBPath) { + if err := os.MkdirAll(cfg.DBPath, 0700); err != nil { return nil, err } } - db, err = Create(BoltBackendName, dbFilePath, noFreeListSync) + db, err = Create(BoltBackendName, dbFilePath, cfg.NoFreelistSync) } else { - db, err = Open(BoltBackendName, dbFilePath, noFreeListSync) + db, err = Open(BoltBackendName, dbFilePath, cfg.NoFreelistSync) } if err != nil { @@ -57,7 +86,11 @@ func GetTestBackend(path, name string) (Backend, func(), error) { empty := func() {} if TestBackend == BoltBackendName { - db, err := GetBoltBackend(path, name, true) + db, err := GetBoltBackend(&BoltBackendConfig{ + DBPath: path, + DBFileName: name, + NoFreelistSync: true, + }) if err != nil { return nil, nil, err } diff --git a/channeldb/options.go b/channeldb/options.go index b84dd199..c9144650 100644 --- a/channeldb/options.go +++ b/channeldb/options.go @@ -1,6 +1,11 @@ package channeldb -import "github.com/lightningnetwork/lnd/clock" +import ( + "time" + + "github.com/lightningnetwork/lnd/channeldb/kvdb" + "github.com/lightningnetwork/lnd/clock" +) const ( // DefaultRejectCacheSize is the default number of rejectCacheEntries to @@ -16,6 +21,8 @@ const ( // Options holds parameters for tuning and customizing a channeldb.DB. type Options struct { + kvdb.BoltBackendConfig + // RejectCacheSize is the maximum number of rejectCacheEntries to hold // in the rejection cache. RejectCacheSize int @@ -24,11 +31,6 @@ type Options struct { // channel cache. ChannelCacheSize int - // NoFreelistSync, if true, prevents the database from syncing its - // freelist to disk, resulting in improved performance at the expense of - // increased startup time. - NoFreelistSync bool - // clock is the time source used by the database. clock clock.Clock @@ -40,9 +42,13 @@ type Options struct { // DefaultOptions returns an Options populated with default values. func DefaultOptions() Options { return Options{ + BoltBackendConfig: kvdb.BoltBackendConfig{ + NoFreelistSync: true, + AutoCompact: false, + AutoCompactMinAge: kvdb.DefaultBoltAutoCompactMinAge, + }, RejectCacheSize: DefaultRejectCacheSize, ChannelCacheSize: DefaultChannelCacheSize, - NoFreelistSync: true, clock: clock.NewDefaultClock(), } } @@ -71,6 +77,21 @@ func OptionSetSyncFreelist(b bool) OptionModifier { } } +// OptionAutoCompact turns on automatic database compaction on startup. +func OptionAutoCompact() OptionModifier { + return func(o *Options) { + o.AutoCompact = true + } +} + +// OptionAutoCompactMinAge sets the minimum age for automatic database +// compaction. +func OptionAutoCompactMinAge(minAge time.Duration) OptionModifier { + return func(o *Options) { + o.AutoCompactMinAge = minAge + } +} + // OptionClock sets a non-default clock dependency. func OptionClock(clock clock.Clock) OptionModifier { return func(o *Options) { diff --git a/lncfg/db.go b/lncfg/db.go index 3b37ba39..2f0eade5 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -83,9 +83,13 @@ func (db *DB) GetBackends(ctx context.Context, dbPath string, } } - localDB, err = kvdb.GetBoltBackend( - dbPath, dbName, !db.Bolt.SyncFreelist, - ) + localDB, err = kvdb.GetBoltBackend(&kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: dbName, + NoFreelistSync: !db.Bolt.SyncFreelist, + AutoCompact: db.Bolt.AutoCompact, + AutoCompactMinAge: db.Bolt.AutoCompactMinAge, + }) if err != nil { return nil, err } diff --git a/lnd.go b/lnd.go index 073ae81e..43692ccc 100644 --- a/lnd.go +++ b/lnd.go @@ -1353,8 +1353,9 @@ func initializeDatabases(ctx context.Context, "minutes...") if cfg.DB.Backend == lncfg.BoltBackend { - ltndLog.Infof("Opening bbolt database, sync_freelist=%v", - cfg.DB.Bolt.SyncFreelist) + ltndLog.Infof("Opening bbolt database, sync_freelist=%v, "+ + "auto_compact=%v", cfg.DB.Bolt.SyncFreelist, + cfg.DB.Bolt.AutoCompact) } startOpenTime := time.Now()