9992e5c0b5
This refactor changes how we instantiate the etcd based Backend to allow separate initialization and startup for the embedded instance.
269 lines
8.4 KiB
Go
269 lines
8.4 KiB
Go
package kvdb
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
_ "github.com/btcsuite/btcwallet/walletdb/bdb" // Import to register backend.
|
|
)
|
|
|
|
const (
|
|
// DefaultTempDBFileName is the default name of the temporary bolt DB
|
|
// file that we'll use to atomically compact the primary DB file on
|
|
// startup.
|
|
DefaultTempDBFileName = "temp-dont-use.db"
|
|
|
|
// LastCompactionFileNameSuffix is the suffix we append to the file name
|
|
// of a database file to record the timestamp when the last compaction
|
|
// occurred.
|
|
LastCompactionFileNameSuffix = ".last-compacted"
|
|
)
|
|
|
|
var (
|
|
byteOrder = binary.BigEndian
|
|
)
|
|
|
|
// fileExists returns true if the file exists, and false otherwise.
|
|
func fileExists(path string) bool {
|
|
if _, err := os.Stat(path); err != nil {
|
|
if os.IsNotExist(err) {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// BoltBackendConfig is a struct that holds settings specific to the bolt
|
|
// database backend.
|
|
type BoltBackendConfig struct {
|
|
// DBPath is the directory path in which the database file should be
|
|
// stored.
|
|
DBPath string
|
|
|
|
// DBFileName is the name of the database file.
|
|
DBFileName string
|
|
|
|
// NoFreelistSync, if true, prevents the database from syncing its
|
|
// freelist to disk, resulting in improved performance at the expense of
|
|
// increased startup time.
|
|
NoFreelistSync bool
|
|
|
|
// AutoCompact specifies if a Bolt based database backend should be
|
|
// automatically compacted on startup (if the minimum age of the
|
|
// database file is reached). This will require additional disk space
|
|
// for the compacted copy of the database but will result in an overall
|
|
// lower database size after the compaction.
|
|
AutoCompact bool
|
|
|
|
// AutoCompactMinAge specifies the minimum time that must have passed
|
|
// since a bolt database file was last compacted for the compaction to
|
|
// be considered again.
|
|
AutoCompactMinAge time.Duration
|
|
|
|
// DBTimeout specifies the timeout value to use when opening the wallet
|
|
// database.
|
|
DBTimeout time.Duration
|
|
}
|
|
|
|
// GetBoltBackend opens (or creates if doesn't exits) a bbolt backed database
|
|
// and returns a kvdb.Backend wrapping it.
|
|
func GetBoltBackend(cfg *BoltBackendConfig) (Backend, error) {
|
|
dbFilePath := filepath.Join(cfg.DBPath, cfg.DBFileName)
|
|
|
|
// Is this a new database?
|
|
if !fileExists(dbFilePath) {
|
|
if !fileExists(cfg.DBPath) {
|
|
if err := os.MkdirAll(cfg.DBPath, 0700); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
return Create(
|
|
BoltBackendName, dbFilePath,
|
|
cfg.NoFreelistSync, cfg.DBTimeout,
|
|
)
|
|
}
|
|
|
|
// This is an existing database. We might want to compact it on startup
|
|
// to free up some space.
|
|
if cfg.AutoCompact {
|
|
if err := compactAndSwap(cfg); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
return Open(
|
|
BoltBackendName, dbFilePath,
|
|
cfg.NoFreelistSync, cfg.DBTimeout,
|
|
)
|
|
}
|
|
|
|
// compactAndSwap will attempt to write a new temporary DB file to disk with
|
|
// the compacted database content, then atomically swap (via rename) the old
|
|
// file for the new file by updating the name of the new file to the old.
|
|
func compactAndSwap(cfg *BoltBackendConfig) error {
|
|
sourceName := cfg.DBFileName
|
|
|
|
// If the main DB file isn't set, then we can't proceed.
|
|
if sourceName == "" {
|
|
return fmt.Errorf("cannot compact DB with empty name")
|
|
}
|
|
sourceFilePath := filepath.Join(cfg.DBPath, sourceName)
|
|
tempDestFilePath := filepath.Join(cfg.DBPath, DefaultTempDBFileName)
|
|
|
|
// Let's find out how long ago the last compaction of the source file
|
|
// occurred and possibly skip compacting it again now.
|
|
lastCompactionDate, err := lastCompactionDate(sourceFilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot determine last compaction date of "+
|
|
"source DB file: %v", err)
|
|
}
|
|
compactAge := time.Since(lastCompactionDate)
|
|
if cfg.AutoCompactMinAge != 0 && compactAge <= cfg.AutoCompactMinAge {
|
|
log.Infof("Not compacting database file at %v, it was last "+
|
|
"compacted at %v (%v ago), min age is set to %v",
|
|
sourceFilePath, lastCompactionDate,
|
|
compactAge.Truncate(time.Second), cfg.AutoCompactMinAge)
|
|
return nil
|
|
}
|
|
|
|
log.Infof("Compacting database file at %v", sourceFilePath)
|
|
|
|
// If the old temporary DB file still exists, then we'll delete it
|
|
// before proceeding.
|
|
if _, err := os.Stat(tempDestFilePath); err == nil {
|
|
log.Infof("Found old temp DB @ %v, removing before swap",
|
|
tempDestFilePath)
|
|
|
|
err = os.Remove(tempDestFilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to remove old temp DB file: "+
|
|
"%v", err)
|
|
}
|
|
}
|
|
|
|
// Now that we know the staging area is clear, we'll create the new
|
|
// temporary DB file and close it before we write the new DB to it.
|
|
tempFile, err := os.Create(tempDestFilePath)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to create temp DB file: %v", err)
|
|
}
|
|
if err := tempFile.Close(); err != nil {
|
|
return fmt.Errorf("unable to close file: %v", err)
|
|
}
|
|
|
|
// With the file created, we'll start the compaction and remove the
|
|
// temporary file all together once this method exits.
|
|
defer func() {
|
|
// This will only succeed if the rename below fails. If the
|
|
// compaction is successful, the file won't exist on exit
|
|
// anymore so no need to log an error here.
|
|
_ = os.Remove(tempDestFilePath)
|
|
}()
|
|
c := &compacter{
|
|
srcPath: sourceFilePath,
|
|
dstPath: tempDestFilePath,
|
|
dbTimeout: cfg.DBTimeout,
|
|
}
|
|
initialSize, newSize, err := c.execute()
|
|
if err != nil {
|
|
return fmt.Errorf("error during compact: %v", err)
|
|
}
|
|
|
|
log.Infof("DB compaction of %v successful, %d -> %d bytes (gain=%.2fx)",
|
|
sourceFilePath, initialSize, newSize,
|
|
float64(initialSize)/float64(newSize))
|
|
|
|
// We try to store the current timestamp in a file with the suffix
|
|
// .last-compacted so we can figure out how long ago the last compaction
|
|
// was. But since this shouldn't fail the compaction process itself, we
|
|
// only log the error. Worst case if this file cannot be written is that
|
|
// we compact on every startup.
|
|
err = updateLastCompactionDate(sourceFilePath)
|
|
if err != nil {
|
|
log.Warnf("Could not update last compaction timestamp in "+
|
|
"%s%s: %v", sourceFilePath,
|
|
LastCompactionFileNameSuffix, err)
|
|
}
|
|
|
|
log.Infof("Swapping old DB file from %v to %v", tempDestFilePath,
|
|
sourceFilePath)
|
|
|
|
// Finally, we'll attempt to atomically rename the temporary file to
|
|
// the main back up file. If this succeeds, then we'll only have a
|
|
// single file on disk once this method exits.
|
|
return os.Rename(tempDestFilePath, sourceFilePath)
|
|
}
|
|
|
|
// lastCompactionDate returns the date the given database file was last
|
|
// compacted or a zero time.Time if no compaction was recorded before. The
|
|
// compaction date is read from a file in the same directory and with the same
|
|
// name as the DB file, but with the suffix ".last-compacted".
|
|
func lastCompactionDate(dbFile string) (time.Time, error) {
|
|
zeroTime := time.Unix(0, 0)
|
|
|
|
tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix)
|
|
if !fileExists(tsFile) {
|
|
return zeroTime, nil
|
|
}
|
|
|
|
tsBytes, err := ioutil.ReadFile(tsFile)
|
|
if err != nil {
|
|
return zeroTime, err
|
|
}
|
|
|
|
tsNano := byteOrder.Uint64(tsBytes)
|
|
return time.Unix(0, int64(tsNano)), nil
|
|
}
|
|
|
|
// updateLastCompactionDate stores the current time as a timestamp in a file
|
|
// in the same directory and with the same name as the DB file, but with the
|
|
// suffix ".last-compacted".
|
|
func updateLastCompactionDate(dbFile string) error {
|
|
var tsBytes [8]byte
|
|
byteOrder.PutUint64(tsBytes[:], uint64(time.Now().UnixNano()))
|
|
|
|
tsFile := fmt.Sprintf("%s%s", dbFile, LastCompactionFileNameSuffix)
|
|
return ioutil.WriteFile(tsFile, tsBytes[:], 0600)
|
|
}
|
|
|
|
// GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd
|
|
// backed database (for testing), and returns a kvdb.Backend and a cleanup
|
|
// func. Whether to create/open bbolt or embedded etcd database is based
|
|
// on the TestBackend constant which is conditionally compiled with build tag.
|
|
// The passed path is used to hold all db files, while the name is only used
|
|
// for bbolt.
|
|
func GetTestBackend(path, name string) (Backend, func(), error) {
|
|
empty := func() {}
|
|
|
|
if TestBackend == BoltBackendName {
|
|
db, err := GetBoltBackend(&BoltBackendConfig{
|
|
DBPath: path,
|
|
DBFileName: name,
|
|
NoFreelistSync: true,
|
|
DBTimeout: DefaultDBTimeout,
|
|
})
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
return db, empty, nil
|
|
} else if TestBackend == EtcdBackendName {
|
|
etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0)
|
|
if err != nil {
|
|
return nil, empty, err
|
|
}
|
|
backend, err := Open(
|
|
EtcdBackendName, context.TODO(), etcdConfig,
|
|
)
|
|
return backend, cancel, err
|
|
}
|
|
|
|
return nil, nil, fmt.Errorf("unknown backend")
|
|
}
|