From 6131a53eb49ed943a16a8f1ce0cc77e2bfc3e7b2 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:23 +0100 Subject: [PATCH 1/8] lncfg+channeldb: add autocompact flags to BoltConfig --- channeldb/kvdb/config.go | 29 +++++++++++++++++++++-------- lncfg/db.go | 4 +++- sample-lnd.conf | 12 ++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/channeldb/kvdb/config.go b/channeldb/kvdb/config.go index 179fde78..ccd1c1d7 100644 --- a/channeldb/kvdb/config.go +++ b/channeldb/kvdb/config.go @@ -1,18 +1,31 @@ package kvdb -// BoltBackendName is the name of the backend that should be passed into -// kvdb.Create to initialize a new instance of kvdb.Backend backed by a live -// instance of bbolt. -const BoltBackendName = "bdb" +import "time" -// EtcdBackendName is the name of the backend that should be passed into -// kvdb.Create to initialize a new instance of kvdb.Backend backed by a live -// instance of etcd. -const EtcdBackendName = "etcd" +const ( + // BoltBackendName is the name of the backend that should be passed into + // kvdb.Create to initialize a new instance of kvdb.Backend backed by a + // live instance of bbolt. + BoltBackendName = "bdb" + + // EtcdBackendName is the name of the backend that should be passed into + // kvdb.Create to initialize a new instance of kvdb.Backend backed by a + // live instance of etcd. + EtcdBackendName = "etcd" + + // DefaultBoltAutoCompactMinAge is the default minimum time that must + // have passed since a bolt database file was last compacted for the + // compaction to be considered again. + DefaultBoltAutoCompactMinAge = time.Hour * 24 * 7 +) // BoltConfig holds bolt configuration. type BoltConfig struct { SyncFreelist bool `long:"nofreelistsync" description:"Whether the databases used within lnd should sync their freelist to disk. This is disabled by default resulting in improved memory performance during operation, but with an increase in startup time."` + + AutoCompact bool `long:"auto-compact" description:"Whether the databases used within lnd should automatically be compacted on every startup (and if the database has the configured minimum age). This is disabled by default because it requires additional disk space to be available during the compaction that is freed afterwards. In general compaction leads to smaller database files."` + + AutoCompactMinAge time.Duration `long:"auto-compact-min-age" description:"How long ago the last compaction of a database file must be for it to be considered for auto compaction again. Can be set to 0 to compact on every startup."` } // EtcdConfig holds etcd configuration. diff --git a/lncfg/db.go b/lncfg/db.go index cb6bcb24..3b37ba39 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -26,7 +26,9 @@ type DB struct { func DefaultDB() *DB { return &DB{ Backend: BoltBackend, - Bolt: &kvdb.BoltConfig{}, + Bolt: &kvdb.BoltConfig{ + AutoCompactMinAge: kvdb.DefaultBoltAutoCompactMinAge, + }, } } diff --git a/sample-lnd.conf b/sample-lnd.conf index f2bc5486..0a6ea722 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -928,3 +928,15 @@ litecoin.node=ltcd [bolt] ; If true, prevents the database from syncing its freelist to disk. ; db.bolt.nofreelistsync=1 + +; Whether the databases used within lnd should automatically be compacted on +; every startup (and if the database has the configured minimum age). This is +; disabled by default because it requires additional disk space to be available +; during the compaction that is freed afterwards. In general compaction leads to +; smaller database files. +; db.bolt.auto-compact=true + +; How long ago the last compaction of a database file must be for it to be +; considered for auto compaction again. Can be set to 0 to compact on every +; startup. (default: 168h) +; db.bolt.auto-compact-min-age=0 From f8907fdb47daaab90923827d2b9d01eeaf235776 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:25 +0100 Subject: [PATCH 2/8] multi: add AutoCompact option to bolt backend With this commit we thread the new AutoCompact flags all the way through to the bolt backend. --- channeldb/db.go | 8 ++++++- channeldb/kvdb/backend.go | 47 +++++++++++++++++++++++++++++++++------ channeldb/options.go | 35 +++++++++++++++++++++++------ lncfg/db.go | 10 ++++++--- lnd.go | 5 +++-- 5 files changed, 85 insertions(+), 20 deletions(-) diff --git a/channeldb/db.go b/channeldb/db.go index 018c01b2..465003c4 100644 --- a/channeldb/db.go +++ b/channeldb/db.go @@ -238,7 +238,13 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) { modifier(&opts) } - backend, err := kvdb.GetBoltBackend(dbPath, dbName, opts.NoFreelistSync) + backend, err := kvdb.GetBoltBackend(&kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: dbName, + NoFreelistSync: opts.NoFreelistSync, + AutoCompact: opts.AutoCompact, + AutoCompactMinAge: opts.AutoCompactMinAge, + }) if err != nil { return nil, err } diff --git a/channeldb/kvdb/backend.go b/channeldb/kvdb/backend.go index 5f710ed9..0acedcb6 100644 --- a/channeldb/kvdb/backend.go +++ b/channeldb/kvdb/backend.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "time" _ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend. ) @@ -19,25 +20,53 @@ func fileExists(path string) bool { return true } +// BoltBackendConfig is a struct that holds settings specific to the bolt +// database backend. +type BoltBackendConfig struct { + // DBPath is the directory path in which the database file should be + // stored. + DBPath string + + // DBFileName is the name of the database file. + DBFileName string + + // NoFreelistSync, if true, prevents the database from syncing its + // freelist to disk, resulting in improved performance at the expense of + // increased startup time. + NoFreelistSync bool + + // AutoCompact specifies if a Bolt based database backend should be + // automatically compacted on startup (if the minimum age of the + // database file is reached). This will require additional disk space + // for the compacted copy of the database but will result in an overall + // lower database size after the compaction. + AutoCompact bool + + // AutoCompactMinAge specifies the minimum time that must have passed + // since a bolt database file was last compacted for the compaction to + // be considered again. + AutoCompactMinAge time.Duration +} + // GetBoltBackend opens (or creates if doesn't exits) a bbolt // backed database and returns a kvdb.Backend wrapping it. -func GetBoltBackend(path, name string, noFreeListSync bool) (Backend, error) { - dbFilePath := filepath.Join(path, name) +func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) { + dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName) var ( db Backend err error ) if !fileExists(dbFilePath) { - if !fileExists(path) { - if err := os.MkdirAll(path, 0700); err != nil { + if !fileExists(cfg.DBPath) { + if err := os.MkdirAll(cfg.DBPath, 0700); err != nil { return nil, err } } - db, err = Create(BoltBackendName, dbFilePath, noFreeListSync) + db, err = Create(BoltBackendName, dbFilePath, cfg.NoFreelistSync) } else { - db, err = Open(BoltBackendName, dbFilePath, noFreeListSync) + db, err = Open(BoltBackendName, dbFilePath, cfg.NoFreelistSync) } if err != nil { @@ -57,7 +86,11 @@ func GetTestBackend(path, name string) (Backend, func(), error) { empty := func() {} if TestBackend == BoltBackendName { - db, err := GetBoltBackend(path, name, true) + db, err := GetBoltBackend(&BoltBackendConfig{ + DBPath: path, + DBFileName: name, + NoFreelistSync: true, + }) if err != nil { return nil, nil, err } diff --git a/channeldb/options.go b/channeldb/options.go index b84dd199..c9144650 100644 --- a/channeldb/options.go +++ b/channeldb/options.go @@ -1,6 +1,11 @@ package channeldb -import "github.com/lightningnetwork/lnd/clock" +import ( + "time" + + "github.com/lightningnetwork/lnd/channeldb/kvdb" + "github.com/lightningnetwork/lnd/clock" +) const ( // DefaultRejectCacheSize is the default number of rejectCacheEntries to @@ -16,6 +21,8 @@ const ( // Options holds parameters for tuning and customizing a channeldb.DB. type Options struct { + kvdb.BoltBackendConfig + // RejectCacheSize is the maximum number of rejectCacheEntries to hold // in the rejection cache. RejectCacheSize int @@ -24,11 +31,6 @@ type Options struct { // channel cache. ChannelCacheSize int - // NoFreelistSync, if true, prevents the database from syncing its - // freelist to disk, resulting in improved performance at the expense of - // increased startup time. - NoFreelistSync bool - // clock is the time source used by the database. clock clock.Clock @@ -40,9 +42,13 @@ type Options struct { // DefaultOptions returns an Options populated with default values. func DefaultOptions() Options { return Options{ + BoltBackendConfig: kvdb.BoltBackendConfig{ + NoFreelistSync: true, + AutoCompact: false, + AutoCompactMinAge: kvdb.DefaultBoltAutoCompactMinAge, + }, RejectCacheSize: DefaultRejectCacheSize, ChannelCacheSize: DefaultChannelCacheSize, - NoFreelistSync: true, clock: clock.NewDefaultClock(), } } @@ -71,6 +77,21 @@ func OptionSetSyncFreelist(b bool) OptionModifier { } } +// OptionAutoCompact turns on automatic database compaction on startup. +func OptionAutoCompact() OptionModifier { + return func(o *Options) { + o.AutoCompact = true + } +} + +// OptionAutoCompactMinAge sets the minimum age for automatic database +// compaction. +func OptionAutoCompactMinAge(minAge time.Duration) OptionModifier { + return func(o *Options) { + o.AutoCompactMinAge = minAge + } +} + // OptionClock sets a non-default clock dependency. func OptionClock(clock clock.Clock) OptionModifier { return func(o *Options) { diff --git a/lncfg/db.go b/lncfg/db.go index 3b37ba39..2f0eade5 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -83,9 +83,13 @@ func (db *DB) GetBackends(ctx context.Context, dbPath string, } } - localDB, err = kvdb.GetBoltBackend( - dbPath, dbName, !db.Bolt.SyncFreelist, - ) + localDB, err = kvdb.GetBoltBackend(&kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: dbName, + NoFreelistSync: !db.Bolt.SyncFreelist, + AutoCompact: db.Bolt.AutoCompact, + AutoCompactMinAge: db.Bolt.AutoCompactMinAge, + }) if err != nil { return nil, err } diff --git a/lnd.go b/lnd.go index 073ae81e..43692ccc 100644 --- a/lnd.go +++ b/lnd.go @@ -1353,8 +1353,9 @@ func initializeDatabases(ctx context.Context, "minutes...") if cfg.DB.Backend == lncfg.BoltBackend { - ltndLog.Infof("Opening bbolt database, sync_freelist=%v", - cfg.DB.Bolt.SyncFreelist) + ltndLog.Infof("Opening bbolt database, sync_freelist=%v, "+ + "auto_compact=%v", cfg.DB.Bolt.SyncFreelist, + cfg.DB.Bolt.AutoCompact) } startOpenTime := time.Now() From 35c1fad517294c178072e2a92d0763726fc432b2 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:26 +0100 Subject: [PATCH 3/8] server+healthcheck: rename function, add absolute disk space function With this commit we rename the existing AvailableDiskSpace function to its correct name AvailableDiskSpaceRatio as it only returns a ratio. We then go ahead and add a new function that returns the actual number of free bytes available on a file system. This also fixes some comments and always returns an error instead of panicking. --- healthcheck/diskcheck.go | 19 +++++++++++++++++-- healthcheck/diskcheck_netbsd.go | 18 +++++++++++++++--- healthcheck/diskcheck_openbsd.go | 18 +++++++++++++++--- healthcheck/diskcheck_solaris.go | 18 +++++++++++++++--- healthcheck/diskcheck_windows.go | 22 ++++++++++++++++++---- server.go | 4 +++- 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/healthcheck/diskcheck.go b/healthcheck/diskcheck.go index ed230d27..7f30b154 100644 --- a/healthcheck/diskcheck.go +++ b/healthcheck/diskcheck.go @@ -4,8 +4,9 @@ package healthcheck import "syscall" -// AvailableDiskSpace returns ratio of available disk space to total capacity. -func AvailableDiskSpace(path string) (float64, error) { +// AvailableDiskSpaceRatio returns ratio of available disk space to total +// capacity. +func AvailableDiskSpaceRatio(path string) (float64, error) { s := syscall.Statfs_t{} err := syscall.Statfs(path, &s) if err != nil { @@ -16,3 +17,17 @@ func AvailableDiskSpace(path string) (float64, error) { // free blocks. return float64(s.Bfree) / float64(s.Blocks), nil } + +// AvailableDiskSpace returns the available disk space in bytes of the given +// file system. +func AvailableDiskSpace(path string) (uint64, error) { + s := syscall.Statfs_t{} + err := syscall.Statfs(path, &s) + if err != nil { + return 0, err + } + + // Some OSes have s.Bavail defined as int64, others as uint64, so we + // need the explicit type conversion here. + return uint64(s.Bavail) * uint64(s.Bsize), nil // nolint:unconvert +} diff --git a/healthcheck/diskcheck_netbsd.go b/healthcheck/diskcheck_netbsd.go index d44330b7..a70e9c35 100644 --- a/healthcheck/diskcheck_netbsd.go +++ b/healthcheck/diskcheck_netbsd.go @@ -2,9 +2,9 @@ package healthcheck import "golang.org/x/sys/unix" -// AvailableDiskSpace returns ratio of available disk space to total capacity -// for solaris. -func AvailableDiskSpace(path string) (float64, error) { +// AvailableDiskSpaceRatio returns ratio of available disk space to total +// capacity for netbsd. +func AvailableDiskSpaceRatio(path string) (float64, error) { s := unix.Statvfs_t{} err := unix.Statvfs(path, &s) if err != nil { @@ -15,3 +15,15 @@ func AvailableDiskSpace(path string) (float64, error) { // free blocks. return float64(s.Bfree) / float64(s.Blocks), nil } + +// AvailableDiskSpace returns the available disk space in bytes of the given +// file system for netbsd. +func AvailableDiskSpace(path string) (uint64, error) { + s := unix.Statvfs_t{} + err := unix.Statvfs(path, &s) + if err != nil { + return 0, err + } + + return s.Bavail * uint64(s.Bsize), nil +} diff --git a/healthcheck/diskcheck_openbsd.go b/healthcheck/diskcheck_openbsd.go index 4738db9a..b7538ff9 100644 --- a/healthcheck/diskcheck_openbsd.go +++ b/healthcheck/diskcheck_openbsd.go @@ -2,9 +2,9 @@ package healthcheck import "golang.org/x/sys/unix" -// AvailableDiskSpace returns ratio of available disk space to total capacity -// for solaris. -func AvailableDiskSpace(path string) (float64, error) { +// AvailableDiskSpaceRatio returns ratio of available disk space to total +// capacity for openbsd. +func AvailableDiskSpaceRatio(path string) (float64, error) { s := unix.Statfs_t{} err := unix.Statfs(path, &s) if err != nil { @@ -15,3 +15,15 @@ func AvailableDiskSpace(path string) (float64, error) { // free blocks. return float64(s.F_bfree) / float64(s.F_blocks), nil } + +// AvailableDiskSpace returns the available disk space in bytes of the given +// file system for openbsd. +func AvailableDiskSpace(path string) (uint64, error) { + s := unix.Statfs_t{} + err := unix.Statfs(path, &s) + if err != nil { + return 0, err + } + + return uint64(s.F_bavail) * uint64(s.F_bsize), nil +} diff --git a/healthcheck/diskcheck_solaris.go b/healthcheck/diskcheck_solaris.go index d44330b7..32d7992f 100644 --- a/healthcheck/diskcheck_solaris.go +++ b/healthcheck/diskcheck_solaris.go @@ -2,9 +2,9 @@ package healthcheck import "golang.org/x/sys/unix" -// AvailableDiskSpace returns ratio of available disk space to total capacity -// for solaris. -func AvailableDiskSpace(path string) (float64, error) { +// AvailableDiskSpaceRatio returns ratio of available disk space to total +// capacity for solaris. +func AvailableDiskSpaceRatio(path string) (float64, error) { s := unix.Statvfs_t{} err := unix.Statvfs(path, &s) if err != nil { @@ -15,3 +15,15 @@ func AvailableDiskSpace(path string) (float64, error) { // free blocks. return float64(s.Bfree) / float64(s.Blocks), nil } + +// AvailableDiskSpace returns the available disk space in bytes of the given +// file system for solaris. +func AvailableDiskSpace(path string) (uint64, error) { + s := unix.Statvfs_t{} + err := unix.Statvfs(path, &s) + if err != nil { + return 0, err + } + + return s.Bavail * uint64(s.Bsize), nil +} diff --git a/healthcheck/diskcheck_windows.go b/healthcheck/diskcheck_windows.go index 7fed088b..a999cf11 100644 --- a/healthcheck/diskcheck_windows.go +++ b/healthcheck/diskcheck_windows.go @@ -2,16 +2,30 @@ package healthcheck import "golang.org/x/sys/windows" -// AvailableDiskSpace returns ratio of available disk space to total capacity -// for windows. -func AvailableDiskSpace(path string) (float64, error) { +// AvailableDiskSpaceRatio returns ratio of available disk space to total +// capacity for windows. +func AvailableDiskSpaceRatio(path string) (float64, error) { var free, total, avail uint64 pathPtr, err := windows.UTF16PtrFromString(path) if err != nil { - panic(err) + return 0, err } err = windows.GetDiskFreeSpaceEx(pathPtr, &free, &total, &avail) return float64(avail) / float64(total), nil } + +// AvailableDiskSpace returns the available disk space in bytes of the given +// file system for windows. +func AvailableDiskSpace(path string) (uint64, error) { + var free, total, avail uint64 + + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + err = windows.GetDiskFreeSpaceEx(pathPtr, &free, &total, &avail) + + return avail, nil +} diff --git a/server.go b/server.go index a20d6cf5..5858909c 100644 --- a/server.go +++ b/server.go @@ -1302,7 +1302,9 @@ func newServer(cfg *Config, listenAddrs []net.Addr, diskCheck := healthcheck.NewObservation( "disk space", func() error { - free, err := healthcheck.AvailableDiskSpace(cfg.LndDir) + free, err := healthcheck.AvailableDiskSpaceRatio( + cfg.LndDir, + ) if err != nil { return err } From 505be0d8bca78ba760426f3a194e9c083633d3ca Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:28 +0100 Subject: [PATCH 4/8] channeldb: implement compaction This commit adds the compaction feature of the bbolt compact to our bolt backend. This will try to open the DB in read-only mode and create a compacted copy in a temporary file. Once the compaction was successful, the temporary file and the old source file are swapped atomically. --- channeldb/kvdb/backend.go | 96 +++++++++++-- channeldb/kvdb/bolt_compact.go | 246 +++++++++++++++++++++++++++++++++ channeldb/kvdb/log.go | 12 ++ channeldb/log.go | 2 + go.mod | 1 + 5 files changed, 346 insertions(+), 11 deletions(-) create mode 100644 channeldb/kvdb/bolt_compact.go create mode 100644 channeldb/kvdb/log.go diff --git a/channeldb/kvdb/backend.go b/channeldb/kvdb/backend.go index 0acedcb6..6409afda 100644 --- a/channeldb/kvdb/backend.go +++ b/channeldb/kvdb/backend.go @@ -9,6 +9,13 @@ import ( _ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend. ) +const ( + // DefaultTempDBFileName is the default name of the temporary bolt DB + // file that we'll use to atomically compact the primary DB file on + // startup. + DefaultTempDBFileName = "temp-dont-use.db" +) + // fileExists returns true if the file exists, and false otherwise. func fileExists(path string) bool { if _, err := os.Stat(path); err != nil { @@ -48,15 +55,12 @@ type BoltBackendConfig struct { AutoCompactMinAge time.Duration } -// GetBoltBackend opens (or creates if doesn't exits) a bbolt -// backed database and returns a kvdb.Backend wrapping it. +// GetBoltBackend opens (or creates if doesn't exits) a bbolt backed database +// and returns a kvdb.Backend wrapping it. func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) { dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName) - var ( - db Backend - err error - ) + // Is this a new database? if !fileExists(dbFilePath) { if !fileExists(cfg.DBPath) { if err := os.MkdirAll(cfg.DBPath, 0700); err != nil { @@ -64,16 +68,86 @@ func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) { } } - db, err = Create(BoltBackendName, dbFilePath, cfg.NoFreelistSync) - } else { - db, err = Open(BoltBackendName, dbFilePath, cfg.NoFreelistSync) + return Create(BoltBackendName, dbFilePath, cfg.NoFreelistSync) } + // This is an existing database. We might want to compact it on startup + // to free up some space. + if cfg.AutoCompact { + if err := compactAndSwap(cfg); err != nil { + return nil, err + } + } + + return Open(BoltBackendName, dbFilePath, cfg.NoFreelistSync) +} + +// compactAndSwap will attempt to write a new temporary DB file to disk with +// the compacted database content, then atomically swap (via rename) the old +// file for the new file by updating the name of the new file to the old. +func compactAndSwap(cfg *BoltBackendConfig) error { + sourceName := cfg.DBFileName + + // If the main DB file isn't set, then we can't proceed. + if sourceName == "" { + return fmt.Errorf("cannot compact DB with empty name") + } + sourceFilePath := filepath.Join(cfg.DBPath, sourceName) + tempDestFilePath := filepath.Join(cfg.DBPath, DefaultTempDBFileName) + + log.Infof("Compacting database file at %v", sourceFilePath) + + // If the old temporary DB file still exists, then we'll delete it + // before proceeding. + if _, err := os.Stat(tempDestFilePath); err == nil { + log.Infof("Found old temp DB @ %v, removing before swap", + tempDestFilePath) + + err = os.Remove(tempDestFilePath) + if err != nil { + return fmt.Errorf("unable to remove old temp DB file: "+ + "%v", err) + } + } + + // Now that we know the staging area is clear, we'll create the new + // temporary DB file and close it before we write the new DB to it. + tempFile, err := os.Create(tempDestFilePath) if err != nil { - return nil, err + return fmt.Errorf("unable to create temp DB file: %v", err) + } + if err := tempFile.Close(); err != nil { + return fmt.Errorf("unable to close file: %v", err) } - return db, nil + // With the file created, we'll start the compaction and remove the + // temporary file all together once this method exits. + defer func() { + // This will only succeed if the rename below fails. If the + // compaction is successful, the file won't exist on exit + // anymore so no need to log an error here. + _ = os.Remove(tempDestFilePath) + }() + c := &compacter{ + srcPath: sourceFilePath, + dstPath: tempDestFilePath, + } + initialSize, newSize, err := c.execute() + if err != nil { + return fmt.Errorf("error during compact: %v", err) + } + + log.Infof("DB compaction of %v successful, %d -> %d bytes (gain=%.2fx)", + sourceFilePath, initialSize, newSize, + float64(initialSize)/float64(newSize)) + + log.Infof("Swapping old DB file from %v to %v", tempDestFilePath, + sourceFilePath) + + // Finally, we'll attempt to atomically rename the temporary file to + // the main back up file. If this succeeds, then we'll only have a + // single file on disk once this method exits. + return os.Rename(tempDestFilePath, sourceFilePath) } // GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd diff --git a/channeldb/kvdb/bolt_compact.go b/channeldb/kvdb/bolt_compact.go new file mode 100644 index 00000000..da8f2798 --- /dev/null +++ b/channeldb/kvdb/bolt_compact.go @@ -0,0 +1,246 @@ +// The code in this file is an adapted version of the bbolt compact command +// implemented in this file: +// https://github.com/etcd-io/bbolt/blob/master/cmd/bbolt/main.go + +package kvdb + +import ( + "fmt" + "os" + "path" + + "github.com/lightningnetwork/lnd/healthcheck" + "go.etcd.io/bbolt" +) + +const ( + // defaultResultFileSizeMultiplier is the default multiplier we apply to + // the current database size to calculate how big it could possibly get + // after compacting, in case the database is already at its optimal size + // and compaction causes it to grow. This should normally not be the + // case but we really want to avoid not having enough disk space for the + // compaction, so we apply a safety margin of 10%. + defaultResultFileSizeMultiplier = float64(1.1) + + // defaultTxMaxSize is the default maximum number of operations that + // are allowed to be executed in a single transaction. + defaultTxMaxSize = 65536 + + // bucketFillSize is the fill size setting that is used for each new + // bucket that is created in the compacted database. This setting is not + // persisted and is therefore only effective for the compaction itself. + // Because during the compaction we only append data a fill percent of + // 100% is optimal for performance. + bucketFillSize = 1.0 +) + +type compacter struct { + srcPath string + dstPath string + txMaxSize int64 +} + +// execute opens the source and destination databases and then compacts the +// source into destination and returns the size of both files as a result. +func (cmd *compacter) execute() (int64, int64, error) { + if cmd.txMaxSize == 0 { + cmd.txMaxSize = defaultTxMaxSize + } + + // Ensure source file exists. + fi, err := os.Stat(cmd.srcPath) + if err != nil { + return 0, 0, fmt.Errorf("error determining source database "+ + "size: %v", err) + } + initialSize := fi.Size() + marginSize := float64(initialSize) * defaultResultFileSizeMultiplier + + // Before opening any of the databases, let's first make sure we have + // enough free space on the destination file system to create a full + // copy of the source DB (worst-case scenario if the compaction doesn't + // actually shrink the file size). + destFolder := path.Dir(cmd.dstPath) + freeSpace, err := healthcheck.AvailableDiskSpace(destFolder) + if err != nil { + return 0, 0, fmt.Errorf("error determining free disk space on "+ + "%s: %v", destFolder, err) + } + log.Debugf("Free disk space on compaction destination file system: "+ + "%d bytes", freeSpace) + if freeSpace < uint64(marginSize) { + return 0, 0, fmt.Errorf("could not start compaction, "+ + "destination folder %s only has %d bytes of free disk "+ + "space available while we need at least %d for worst-"+ + "case compaction", destFolder, freeSpace, initialSize) + } + + // Open source database. We open it in read only mode to avoid (and fix) + // possible freelist sync problems. + src, err := bbolt.Open(cmd.srcPath, 0444, &bbolt.Options{ + ReadOnly: true, + }) + if err != nil { + return 0, 0, fmt.Errorf("error opening source database: %v", + err) + } + defer func() { + if err := src.Close(); err != nil { + log.Errorf("Compact error: closing source DB: %v", err) + } + }() + + // Open destination database. + dst, err := bbolt.Open(cmd.dstPath, fi.Mode(), nil) + if err != nil { + return 0, 0, fmt.Errorf("error opening destination database: "+ + "%v", err) + } + defer func() { + if err := dst.Close(); err != nil { + log.Errorf("Compact error: closing dest DB: %v", err) + } + }() + + // Run compaction. + if err := cmd.compact(dst, src); err != nil { + return 0, 0, fmt.Errorf("error running compaction: %v", err) + } + + // Report stats on new size. + fi, err = os.Stat(cmd.dstPath) + if err != nil { + return 0, 0, fmt.Errorf("error determining destination "+ + "database size: %v", err) + } else if fi.Size() == 0 { + return 0, 0, fmt.Errorf("zero db size") + } + + return initialSize, fi.Size(), nil +} + +// compact tries to create a compacted copy of the source database in a new +// destination database. +func (cmd *compacter) compact(dst, src *bbolt.DB) error { + // Commit regularly, or we'll run out of memory for large datasets if + // using one transaction. + var size int64 + tx, err := dst.Begin(true) + if err != nil { + return err + } + defer func() { + _ = tx.Rollback() + }() + + if err := cmd.walk(src, func(keys [][]byte, k, v []byte, seq uint64) error { + // On each key/value, check if we have exceeded tx size. + sz := int64(len(k) + len(v)) + if size+sz > cmd.txMaxSize && cmd.txMaxSize != 0 { + // Commit previous transaction. + if err := tx.Commit(); err != nil { + return err + } + + // Start new transaction. + tx, err = dst.Begin(true) + if err != nil { + return err + } + size = 0 + } + size += sz + + // Create bucket on the root transaction if this is the first + // level. + nk := len(keys) + if nk == 0 { + bkt, err := tx.CreateBucket(k) + if err != nil { + return err + } + if err := bkt.SetSequence(seq); err != nil { + return err + } + return nil + } + + // Create buckets on subsequent levels, if necessary. + b := tx.Bucket(keys[0]) + if nk > 1 { + for _, k := range keys[1:] { + b = b.Bucket(k) + } + } + + // Fill the entire page for best compaction. + b.FillPercent = bucketFillSize + + // If there is no value then this is a bucket call. + if v == nil { + bkt, err := b.CreateBucket(k) + if err != nil { + return err + } + if err := bkt.SetSequence(seq); err != nil { + return err + } + return nil + } + + // Otherwise treat it as a key/value pair. + return b.Put(k, v) + }); err != nil { + return err + } + + return tx.Commit() +} + +// walkFunc is the type of the function called for keys (buckets and "normal" +// values) discovered by Walk. keys is the list of keys to descend to the bucket +// owning the discovered key/value pair k/v. +type walkFunc func(keys [][]byte, k, v []byte, seq uint64) error + +// walk walks recursively the bolt database db, calling walkFn for each key it +// finds. +func (cmd *compacter) walk(db *bbolt.DB, walkFn walkFunc) error { + return db.View(func(tx *bbolt.Tx) error { + return tx.ForEach(func(name []byte, b *bbolt.Bucket) error { + // This will log the top level buckets only to give the + // user some sense of progress. + log.Debugf("Compacting top level bucket %s", name) + + return cmd.walkBucket( + b, nil, name, nil, b.Sequence(), walkFn, + ) + }) + }) +} + +// walkBucket recursively walks through a bucket. +func (cmd *compacter) walkBucket(b *bbolt.Bucket, keyPath [][]byte, k, v []byte, + seq uint64, fn walkFunc) error { + + // Execute callback. + if err := fn(keyPath, k, v, seq); err != nil { + return err + } + + // If this is not a bucket then stop. + if v != nil { + return nil + } + + // Iterate over each child key/value. + keyPath = append(keyPath, k) + return b.ForEach(func(k, v []byte) error { + if v == nil { + bkt := b.Bucket(k) + return cmd.walkBucket( + bkt, keyPath, k, nil, bkt.Sequence(), fn, + ) + } + return cmd.walkBucket(b, keyPath, k, v, b.Sequence(), fn) + }) +} diff --git a/channeldb/kvdb/log.go b/channeldb/kvdb/log.go new file mode 100644 index 00000000..628d48be --- /dev/null +++ b/channeldb/kvdb/log.go @@ -0,0 +1,12 @@ +package kvdb + +import "github.com/btcsuite/btclog" + +// log is a logger that is initialized as disabled. This means the package will +// not perform any logging by default until a logger is set. +var log = btclog.Disabled + +// UseLogger uses a specified Logger to output package logging info. +func UseLogger(logger btclog.Logger) { + log = logger +} diff --git a/channeldb/log.go b/channeldb/log.go index 75ba2a5f..92d8c3e7 100644 --- a/channeldb/log.go +++ b/channeldb/log.go @@ -3,6 +3,7 @@ package channeldb import ( "github.com/btcsuite/btclog" "github.com/lightningnetwork/lnd/build" + "github.com/lightningnetwork/lnd/channeldb/kvdb" mig "github.com/lightningnetwork/lnd/channeldb/migration" "github.com/lightningnetwork/lnd/channeldb/migration12" "github.com/lightningnetwork/lnd/channeldb/migration13" @@ -35,4 +36,5 @@ func UseLogger(logger btclog.Logger) { migration12.UseLogger(logger) migration13.UseLogger(logger) migration16.UseLogger(logger) + kvdb.UseLogger(logger) } diff --git a/go.mod b/go.mod index 464bf779..998b54bf 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02 github.com/urfave/cli v1.18.0 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 go.uber.org/zap v1.14.1 // indirect golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 golang.org/x/net v0.0.0-20191002035440-2ec189313ef0 From a8ef4fc1588bb614360ccb77fb9c1f86fe11fbf3 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:29 +0100 Subject: [PATCH 5/8] channeldb: store and evaluate last compaction time --- channeldb/kvdb/backend.go | 71 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/channeldb/kvdb/backend.go b/channeldb/kvdb/backend.go index 6409afda..e2ac7ae6 100644 --- a/channeldb/kvdb/backend.go +++ b/channeldb/kvdb/backend.go @@ -1,7 +1,9 @@ package kvdb import ( + "encoding/binary" "fmt" + "io/ioutil" "os" "path/filepath" "time" @@ -14,6 +16,15 @@ const ( // file that we'll use to atomically compact the primary DB file on // startup. DefaultTempDBFileName = "temp-dont-use.db" + + // LastCompactionFileNameSuffix is the suffix we append to the file name + // of a database file to record the timestamp when the last compaction + // occurred. + LastCompactionFileNameSuffix = ".last-compacted" +) + +var ( + byteOrder = binary.BigEndian ) // fileExists returns true if the file exists, and false otherwise. @@ -95,6 +106,22 @@ func compactAndSwap(cfg *BoltBackendConfig) error { sourceFilePath := filepath.Join(cfg.DBPath, sourceName) tempDestFilePath := filepath.Join(cfg.DBPath, DefaultTempDBFileName) + // Let's find out how long ago the last compaction of the source file + // occurred and possibly skip compacting it again now. + lastCompactionDate, err := lastCompactionDate(sourceFilePath) + if err != nil { + return fmt.Errorf("cannot determine last compaction date of "+ + "source DB file: %v", err) + } + compactAge := time.Since(lastCompactionDate) + if cfg.AutoCompactMinAge != 0 && compactAge <= cfg.AutoCompactMinAge { + log.Infof("Not compacting database file at %v, it was last "+ + "compacted at %v (%v ago), min age is set to %v", + sourceFilePath, lastCompactionDate, + compactAge.Truncate(time.Second), cfg.AutoCompactMinAge) + return nil + } + log.Infof("Compacting database file at %v", sourceFilePath) // If the old temporary DB file still exists, then we'll delete it @@ -141,6 +168,18 @@ func compactAndSwap(cfg *BoltBackendConfig) error { sourceFilePath, initialSize, newSize, float64(initialSize)/float64(newSize)) + // We try to store the current timestamp in a file with the suffix + // .last-compacted so we can figure out how long ago the last compaction + // was. But since this shouldn't fail the compaction process itself, we + // only log the error. Worst case if this file cannot be written is that + // we compact on every startup. + err = updateLastCompactionDate(sourceFilePath) + if err != nil { + log.Warnf("Could not update last compaction timestamp in "+ + "%s%s: %v", sourceFilePath, + LastCompactionFileNameSuffix, err) + } + log.Infof("Swapping old DB file from %v to %v", tempDestFilePath, sourceFilePath) @@ -150,6 +189,38 @@ func compactAndSwap(cfg *BoltBackendConfig) error { return os.Rename(tempDestFilePath, sourceFilePath) } +// lastCompactionDate returns the date the given database file was last +// compacted or a zero time.Time if no compaction was recorded before. The +// compaction date is read from a file in the same directory and with the same +// name as the DB file, but with the suffix ".last-compacted". +func lastCompactionDate(dbFile string) (time.Time, error) { + zeroTime := time.Unix(0, 0) + + tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix) + if !fileExists(tsFile) { + return zeroTime, nil + } + + tsBytes, err := ioutil.ReadFile(tsFile) + if err != nil { + return zeroTime, err + } + + tsNano := byteOrder.Uint64(tsBytes) + return time.Unix(0, int64(tsNano)), nil +} + +// updateLastCompactionDate stores the current time as a timestamp in a file +// in the same directory and with the same name as the DB file, but with the +// suffix ".last-compacted". +func updateLastCompactionDate(dbFile string) error { + var tsBytes [8]byte + byteOrder.PutUint64(tsBytes[:], uint64(time.Now().UnixNano())) + + tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix) + return ioutil.WriteFile(tsFile, tsBytes[:], 0600) +} + // GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd // backed database (for testing), and returns a kvdb.Backend and a cleanup // func. Whether to create/open bbolt or embedded etcd database is based From 32ee527f6bb2fd6b6f64ceffdaec23b8eae1a78e Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:30 +0100 Subject: [PATCH 6/8] server+htlcswitch: auto compact sphinx DB too --- htlcswitch/decayedlog.go | 20 ++++++++++++------ htlcswitch/decayedlog_test.go | 40 ++++++++++++++++++----------------- server.go | 7 +++--- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/htlcswitch/decayedlog.go b/htlcswitch/decayedlog.go index 59694b07..e9248aa0 100644 --- a/htlcswitch/decayedlog.go +++ b/htlcswitch/decayedlog.go @@ -51,7 +51,7 @@ type DecayedLog struct { started int32 // To be used atomically. stopped int32 // To be used atomically. - dbPath string + cfg *kvdb.BoltBackendConfig db kvdb.Backend @@ -64,16 +64,24 @@ type DecayedLog struct { // NewDecayedLog creates a new DecayedLog, which caches recently seen hash // shared secrets. Entries are evicted as their cltv expires using block epochs // from the given notifier. -func NewDecayedLog(dbPath string, +func NewDecayedLog(dbPath, dbFileName string, boltCfg *kvdb.BoltConfig, notifier chainntnfs.ChainNotifier) *DecayedLog { + cfg := &kvdb.BoltBackendConfig{ + DBPath: dbPath, + DBFileName: dbFileName, + NoFreelistSync: true, + AutoCompact: boltCfg.AutoCompact, + AutoCompactMinAge: boltCfg.AutoCompactMinAge, + } + // Use default path for log database if dbPath == "" { - dbPath = defaultDbDirectory + cfg.DBPath = defaultDbDirectory } return &DecayedLog{ - dbPath: dbPath, + cfg: cfg, notifier: notifier, quit: make(chan struct{}), } @@ -89,9 +97,7 @@ func (d *DecayedLog) Start() error { // Open the boltdb for use. var err error - d.db, err = kvdb.Create( - kvdb.BoltBackendName, d.dbPath, true, - ) + d.db, err = kvdb.GetBoltBackend(d.cfg) if err != nil { return fmt.Errorf("could not open boltdb: %v", err) } diff --git a/htlcswitch/decayedlog_test.go b/htlcswitch/decayedlog_test.go index 961eade4..274ed63f 100644 --- a/htlcswitch/decayedlog_test.go +++ b/htlcswitch/decayedlog_test.go @@ -4,12 +4,12 @@ import ( "crypto/rand" "io/ioutil" "os" - "path/filepath" "testing" "time" sphinx "github.com/lightningnetwork/lightning-onion" "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/lightningnetwork/lnd/channeldb/kvdb" "github.com/lightningnetwork/lnd/lntest/mock" ) @@ -19,17 +19,17 @@ const ( // tempDecayedLogPath creates a new temporary database path to back a single // deccayed log instance. -func tempDecayedLogPath(t *testing.T) string { +func tempDecayedLogPath(t *testing.T) (string, string) { dir, err := ioutil.TempDir("", "decayedlog") if err != nil { t.Fatalf("unable to create temporary decayed log dir: %v", err) } - return filepath.Join(dir, "sphinxreplay.db") + return dir, "sphinxreplay.db" } // startup sets up the DecayedLog and possibly the garbage collector. -func startup(dbPath string, notifier bool) (sphinx.ReplayLog, +func startup(dbPath, dbFileName string, notifier bool) (sphinx.ReplayLog, *mock.ChainNotifier, *sphinx.HashPrefix, error) { var log sphinx.ReplayLog @@ -44,10 +44,12 @@ func startup(dbPath string, notifier bool) (sphinx.ReplayLog, } // Initialize the DecayedLog object - log = NewDecayedLog(dbPath, chainNotifier) + log = NewDecayedLog( + dbPath, dbFileName, &kvdb.BoltConfig{}, chainNotifier, + ) } else { // Initialize the DecayedLog object - log = NewDecayedLog(dbPath, nil) + log = NewDecayedLog(dbPath, dbFileName, &kvdb.BoltConfig{}, nil) } // Open the channeldb (start the garbage collector) @@ -81,9 +83,9 @@ func shutdown(dir string, d sphinx.ReplayLog) { func TestDecayedLogGarbageCollector(t *testing.T) { t.Parallel() - dbPath := tempDecayedLogPath(t) + dbPath, dbFileName := tempDecayedLogPath(t) - d, notifier, hashedSecret, err := startup(dbPath, true) + d, notifier, hashedSecret, err := startup(dbPath, dbFileName, true) if err != nil { t.Fatalf("Unable to start up DecayedLog: %v", err) } @@ -142,9 +144,9 @@ func TestDecayedLogGarbageCollector(t *testing.T) { func TestDecayedLogPersistentGarbageCollector(t *testing.T) { t.Parallel() - dbPath := tempDecayedLogPath(t) + dbPath, dbFileName := tempDecayedLogPath(t) - d, _, hashedSecret, err := startup(dbPath, true) + d, _, hashedSecret, err := startup(dbPath, dbFileName, true) if err != nil { t.Fatalf("Unable to start up DecayedLog: %v", err) } @@ -164,7 +166,7 @@ func TestDecayedLogPersistentGarbageCollector(t *testing.T) { // Shut down DecayedLog and the garbage collector along with it. d.Stop() - d2, notifier2, _, err := startup(dbPath, true) + d2, notifier2, _, err := startup(dbPath, dbFileName, true) if err != nil { t.Fatalf("Unable to restart DecayedLog: %v", err) } @@ -198,9 +200,9 @@ func TestDecayedLogPersistentGarbageCollector(t *testing.T) { func TestDecayedLogInsertionAndDeletion(t *testing.T) { t.Parallel() - dbPath := tempDecayedLogPath(t) + dbPath, dbFileName := tempDecayedLogPath(t) - d, _, hashedSecret, err := startup(dbPath, false) + d, _, hashedSecret, err := startup(dbPath, dbFileName, false) if err != nil { t.Fatalf("Unable to start up DecayedLog: %v", err) } @@ -236,9 +238,9 @@ func TestDecayedLogInsertionAndDeletion(t *testing.T) { func TestDecayedLogStartAndStop(t *testing.T) { t.Parallel() - dbPath := tempDecayedLogPath(t) + dbPath, dbFileName := tempDecayedLogPath(t) - d, _, hashedSecret, err := startup(dbPath, false) + d, _, hashedSecret, err := startup(dbPath, dbFileName, false) if err != nil { t.Fatalf("Unable to start up DecayedLog: %v", err) } @@ -253,7 +255,7 @@ func TestDecayedLogStartAndStop(t *testing.T) { // Shutdown the DecayedLog's channeldb d.Stop() - d2, _, hashedSecret2, err := startup(dbPath, false) + d2, _, hashedSecret2, err := startup(dbPath, dbFileName, false) if err != nil { t.Fatalf("Unable to restart DecayedLog: %v", err) } @@ -280,7 +282,7 @@ func TestDecayedLogStartAndStop(t *testing.T) { // Shutdown the DecayedLog's channeldb d2.Stop() - d3, _, hashedSecret3, err := startup(dbPath, false) + d3, _, hashedSecret3, err := startup(dbPath, dbFileName, false) if err != nil { t.Fatalf("Unable to restart DecayedLog: %v", err) } @@ -302,9 +304,9 @@ func TestDecayedLogStartAndStop(t *testing.T) { func TestDecayedLogStorageAndRetrieval(t *testing.T) { t.Parallel() - dbPath := tempDecayedLogPath(t) + dbPath, dbFileName := tempDecayedLogPath(t) - d, _, hashedSecret, err := startup(dbPath, false) + d, _, hashedSecret, err := startup(dbPath, dbFileName, false) if err != nil { t.Fatalf("Unable to start up DecayedLog: %v", err) } diff --git a/server.go b/server.go index 5858909c..7b6f9b65 100644 --- a/server.go +++ b/server.go @@ -10,7 +10,6 @@ import ( "math/big" prand "math/rand" "net" - "path/filepath" "regexp" "strconv" "sync" @@ -371,10 +370,10 @@ func newServer(cfg *Config, listenAddrs []net.Addr, // Initialize the sphinx router, placing it's persistent replay log in // the same directory as the channel graph database. We don't need to // replicate this data, so we'll store it locally. - sharedSecretPath := filepath.Join( - cfg.localDatabaseDir(), defaultSphinxDbName, + replayLog := htlcswitch.NewDecayedLog( + cfg.localDatabaseDir(), defaultSphinxDbName, cfg.DB.Bolt, + cc.ChainNotifier, ) - replayLog := htlcswitch.NewDecayedLog(sharedSecretPath, cc.ChainNotifier) sphinxRouter := sphinx.NewRouter( nodeKeyECDH, cfg.ActiveNetParams.Params, replayLog, ) From 9bd8784ae8391c8d8000ea001b5a5aa91bee367c Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:21:32 +0100 Subject: [PATCH 7/8] GitHub: run cross compilation for all architectures Because we now have conditionally compiled code that depends on the architecture it is built for, we want to make sure we can build all architectures that we also release. Since GitHub builds are very fast, we can easily do this instead of only compiling for certain select architectures. --- .github/workflows/main.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3ee4fa2b..c31ddad7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -129,12 +129,6 @@ jobs: cross-compile: name: cross compilation runs-on: ubuntu-latest - strategy: - matrix: - build_sys: - - windows-amd64 - - freebsd-amd64 - - solaris-amd64 steps: - name: git checkout uses: actions/checkout@v2 @@ -155,8 +149,8 @@ jobs: with: go-version: '~${{ env.GO_VERSION }}' - - name: build release for architecture - run: make release sys=${{ matrix.build_sys }} + - name: build release for all architectures + run: make release ######################## # mobile compilation From 30c2c0addcee0cff8118b665efb0264dc559680d Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 9 Nov 2020 10:31:00 +0100 Subject: [PATCH 8/8] make: remove arch darwin-386 that is unsupported by go 1.15 As of go version 1.15.x, the darwin-386 architecture is no longer supported. Because we use that go version on Travis to assert all architectures can be built successfully, we have to remove this architecture from the list. --- make/release_flags.mk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/make/release_flags.mk b/make/release_flags.mk index f2d7a2b3..e0058133 100644 --- a/make/release_flags.mk +++ b/make/release_flags.mk @@ -1,8 +1,7 @@ VERSION_TAG = $(shell date +%Y%m%d)-01 VERSION_CHECK = @$(call print, "Building master with date version tag") -BUILD_SYSTEM = darwin-386 \ -darwin-amd64 \ +BUILD_SYSTEM = darwin-amd64 \ dragonfly-amd64 \ freebsd-386 \ freebsd-amd64 \